Fix : 크롤링 봇 추가 패치
This commit is contained in:
parent
726b88f367
commit
651cdcf6f9
75
robots.txt
75
robots.txt
@ -6,6 +6,81 @@ User-agent: Yeti
|
||||
Allow: /
|
||||
Crawl-delay: 1
|
||||
|
||||
# 소셜미디어 크롤러 명시적 허용
|
||||
# Facebook/Meta
|
||||
User-agent: facebookexternalhit/1.1
|
||||
Allow: /
|
||||
User-agent: Facebookbot
|
||||
Allow: /
|
||||
|
||||
# Twitter/X
|
||||
User-agent: Twitterbot
|
||||
Allow: /
|
||||
|
||||
# LinkedIn
|
||||
User-agent: LinkedInBot
|
||||
Allow: /
|
||||
|
||||
# Discord
|
||||
User-agent: Discordbot
|
||||
Allow: /
|
||||
|
||||
# WhatsApp
|
||||
User-agent: WhatsApp
|
||||
Allow: /
|
||||
|
||||
# Telegram
|
||||
User-agent: TelegramBot
|
||||
Allow: /
|
||||
|
||||
# Slack
|
||||
User-agent: Slackbot
|
||||
Allow: /
|
||||
User-agent: Slackbot-LinkExpanding
|
||||
Allow: /
|
||||
|
||||
# Pinterest
|
||||
User-agent: Pinterest
|
||||
Allow: /
|
||||
User-agent: Pinterestbot
|
||||
Allow: /
|
||||
|
||||
# Apple/iMessage
|
||||
User-agent: AppleBot
|
||||
Allow: /
|
||||
|
||||
# Microsoft Teams
|
||||
User-agent: MicrosoftPreview
|
||||
Allow: /
|
||||
|
||||
# KakaoTalk
|
||||
User-agent: KakaoTalk-Scrap
|
||||
Allow: /
|
||||
|
||||
# Line
|
||||
User-agent: Linebot
|
||||
Allow: /
|
||||
|
||||
# Skype
|
||||
User-agent: SkypeUriPreview
|
||||
Allow: /
|
||||
|
||||
# Viber
|
||||
User-agent: ViberBot
|
||||
Allow: /
|
||||
|
||||
# Google (검색 및 소셜)
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
|
||||
# Bing
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
|
||||
# DuckDuckGo
|
||||
User-agent: DuckDuckBot
|
||||
Allow: /
|
||||
|
||||
# 중요한 페이지 우선순위
|
||||
Sitemap: https://minglestudio.co.kr/sitemap.xml
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user