b59961b391cdc4400515082b97ab3998 *DESCRIPTION
243cfc67b7b06ed97d9ffd2a858cc964 *LICENSE
9f1f57da5f924d21b27794345411d194 *NAMESPACE
f426a8b1e5526de390d78e1be45eed10 *NEWS.md
8ea2ae98bd1872538ca6d6046ba64d88 *R/get_robotstxt.R
d2b0b9aecd8be22aae7390aba765853d *R/get_robotstxt_http_get.R
ce9d3cf0f71398746346883c7423078a *R/get_robotstxts.R
e57345372231ac5573a554d205544ad9 *R/guess_domain.R
ad57b0dfc971640f22f80d92abd65ab1 *R/parse_robotstxt.R
8cb05193e001168f8fd76f240d3291b7 *R/paths_allowed.R
75bece4ba04636fe0434b5c2743efc4f *R/paths_allowed_worker_spiderbar.R
e002670688bb09fd0d464f674968fc65 *R/pipe.R
3df4af40a6b8b0073c2c913c2aea1afd *R/print_robotstxt.R
744a71103d4ea2137f919b9bc57962fe *R/print_robotstxt_text.R
1dd2e720f2caba61d07f8605315a6149 *R/remove_domain.R
3d30f215a068dd08efb5ec55378bfa2e *R/robotstxt.R
7d34b0e8417b244318a5e39c5ae010ec *R/rt_cache.R
716845e77f07f6f8f91132581ab79845 *R/rt_get_comments.R
6a0b9aa3b2b5571a8714ab712cb3d0db *R/rt_get_fields.R
ec72b13ccc25816f202e1fd05a25a4fc *R/rt_get_fields_worker.R
40d44e8efb0e17bc3daebdc2bb93dd2c *R/rt_get_useragent.R
9c483bb690f14cc30818c7509f324750 *R/sanitize_path.R
b1f74ca0815a34b1ef3718e05e2ab9ea *R/tools.R
6b654aadf0012745747e84cf3ed036f9 *R/valid_robotstxt.R
a97b6290ca697bc5bd39840dfe9af471 *README.md
bfc0cd0fded6d95141be080c94d551a9 *build/vignette.rds
4bfde16a5fa9f87374ea46998a23c43b *inst/doc/using_robotstxt.R
8ea285b77b146385e74129e581eaff94 *inst/doc/using_robotstxt.Rmd
9aed9e6eff2f4d4813c27ecdf9a689dc *inst/doc/using_robotstxt.html
4d8921c3dc225aa07cd9b3f66cde3aff *inst/robotstxts/allow_single_bot.txt
e2b27c9301b4f24c3b2db3f8886b2dc1 *inst/robotstxts/crawl_delay.txt
8cb7a8b11cc42f1f77aa0670c43071a3 *inst/robotstxts/disallow_all_for_BadBot.txt
d35cb09d5662e686347b00b959c1550f *inst/robotstxts/disallow_all_for_all.txt
542398cc4e150fd288cea549bdca16a4 *inst/robotstxts/disallow_some_for_all.txt
9ca8636f08a496a82ed0a64694d005c8 *inst/robotstxts/disallow_two_at_once.txt
7215ee9c7d9dc229d2921a40e899ec5f *inst/robotstxts/empty.txt
d034a6e564902694ac6b372940786d80 *inst/robotstxts/host.txt
95d8d7433b4cd37b5630cf68500127f0 *inst/robotstxts/robots_amazon.txt
a09430767142d30f771f9726196048e1 *inst/robotstxts/robots_bundestag.txt
7a58b13412a11e1c8650008ae15f2bef *inst/robotstxts/robots_cdc.txt
7a58b13412a11e1c8650008ae15f2bef *inst/robotstxts/robots_cdc2.txt
3b1a4b952779358ff7d6015b0dee0e68 *inst/robotstxts/robots_facebook.txt
c51eaab2c346b3a156c72097e52b10d9 *inst/robotstxts/robots_facebook_unsupported.txt
62dbaccb94d6ab8aa5e178339efbe127 *inst/robotstxts/robots_google.txt
658622dc6bda3645b94e37fa1ea38295 *inst/robotstxts/robots_new_york_times.txt
48a1701be03cf9590a8ef623c5f2e8e6 *inst/robotstxts/robots_pmeissner.txt
67dfdd8b9acd9363417c145388315dab *inst/robotstxts/robots_spiegel.txt
6ed07c3b696c3b4d47e1b3b4a42f5c7f *inst/robotstxts/robots_wikipedia.txt
b03c30fd553878fce698924ee1eca06e *inst/robotstxts/robots_wikipedia_20170706.txt
4637c3108489f0f0e5e283ef16884baf *inst/robotstxts/robots_yahoo.txt
6b8539213a6199601ee9d81c70a6ac31 *inst/robotstxts/selfhtml_Example.txt
2b9b53034510e3b66af8968145fe0a4f *inst/robotstxts/testing_comments.txt
a6f4f6d1bedffc16330a9d11020a0936 *inst/urls.txt
19e6e0845c3c67ee1317523910936e07 *man/get_robotstxt.Rd
80f81d5c39cb8a28e87d1206394a1f32 *man/get_robotstxt_http_get.Rd
4828da0f6a4b5fba6154d8719c7b1e37 *man/get_robotstxts.Rd
1bc0dd9463de688f6dc5022b8408fa4f *man/guess_domain.Rd
6a58cbd3fec1c4a180af5e76794305fc *man/is_valid_robotstxt.Rd
bb2cf9ca030a8efab23b1b232f980400 *man/named_list.Rd
d785fba36311037b573908948b3c107c *man/parse_robotstxt.Rd
2b3257fad2e0c6717858930ab611fca1 *man/paths_allowed.Rd
55780d95a65967df4876b4da3be98545 *man/paths_allowed_worker_spiderbar.Rd
21dcf150608efd6b317d5560114731e3 *man/pipe.Rd
07513817e6f91a93f67ffcc94e689e5b *man/print.robotstxt.Rd
cd914240910650636f6ec6524e5a223f *man/print.robotstxt_text.Rd
af3c208672ffce7a5911e8830a7facd7 *man/remove_domain.Rd
87572edba013abc8e068875d51d1e27d *man/robotstxt.Rd
829dbc0e7dc29c20a176218e8561704a *man/rt_cache.Rd
21e08d07c661487ad4c1f2a67a2c765c *man/rt_get_comments.Rd
3fb293f87ace37db20f97081469edb19 *man/rt_get_fields.Rd
bebcba122f4ccf2d324dd88a475abb8a *man/rt_get_fields_worker.Rd
252430a070d01ed98b385ec8fd38f212 *man/rt_get_rtxt.Rd
10b6a9bf4ed2c9e829d15ff01a768153 *man/rt_get_useragent.Rd
f43aa8ad76ab5c2b2ea2737d5421cfce *man/rt_list_rtxt.Rd
aca99f66c8046ae574c13ebeef6b65e2 *man/sanitize_path.Rd
480e44e1c814faf1886f057b8a375590 *tests/testthat.R
8ae85233dc20826cc031b030ac47d512 *tests/testthat/test_parser.R
45924a0cae3c5a1c81c9da72bf189bf9 *tests/testthat/test_paths_allowed.R
f0d802485eafb5f294ca65195d31ef14 *tests/testthat/test_robotstxt.R
439078b9cea1c5de6925ba475ab67539 *tests/testthat/test_tools.R
8ea285b77b146385e74129e581eaff94 *vignettes/using_robotstxt.Rmd
