{"id":20945,"date":"2023-06-01T23:52:11","date_gmt":"2023-06-01T14:52:11","guid":{"rendered":"http:\/\/www.code-magagine.com\/?p=20945"},"modified":"2023-06-07T23:38:20","modified_gmt":"2023-06-07T14:38:20","slug":"%e3%80%90python%e3%80%91scrapy%e3%81%ae%e6%a7%8b%e6%88%90","status":"publish","type":"post","link":"http:\/\/www.code-magagine.com\/?p=20945","title":{"rendered":"\u3010Python\u3011Scrapy\u306e\u69cb\u6210\u3001\u51e6\u7406\u306e\u6d41\u308c"},"content":{"rendered":"<h2>\u69cb\u6210<\/h2>\n<p><img decoding=\"async\" loading=\"lazy\" class=\"alignnone size-large wp-image-20947\" src=\"http:\/\/www.code-magagine.com\/wp-content\/uploads\/2023\/06\/\u30b9\u30af\u30ea\u30fc\u30f3\u30b7\u30e7\u30c3\u30c8-2023-06-01-23.49.48-1024x496.png\" alt=\"\" width=\"800\" height=\"387\" \/><\/p>\n<h3>Spider<\/h3>\n<p>Scrapy\u3067\u306f\u4e3b\u306b\u30b3\u30fc\u30c9\u306f\u3053\u306e\u30af\u30e9\u30b9\u306b\u8a18\u8ff0\u3092\u3057\u3066\u3044\u304d\u307e\u3059\u3002\u3053\u3053\u306b\u8a18\u8ff0\u3059\u308c\u3070Scrapy\u304c\u3046\u307e\u3044\u3053\u3068\u4ed6\u306e\u4f5c\u696d\u3092\u3084\u3063\u3066\u304f\u308c\u308b\u3088\u3046\u306b\u306a\u308a\u307e\u3059\u3002<\/p>\n<p>\u958b\u767a\u8005\u306fSpider\u3067\u4ee5\u4e0b\u306e\u7269\u3092\u6307\u5b9a\u3057\u307e\u3059\u3002<\/p>\n<ul>\n<li>\u6700\u521d\u306b\u30a2\u30af\u30bb\u30b9\u3059\u308bURL\u3092\u6307\u5b9a<\/li>\n<li>\u30ea\u30f3\u30af\u306e\u305f\u3069\u308a\u65b9\u3092\u6307\u5b9a<\/li>\n<li>\u5fc5\u8981\u306a\u30c7\u30fc\u30bf\u62bd\u51fa\u6cd5\u3092\u6307\u5b9a<\/li>\n<\/ul>\n<h4>Middleware<\/h4>\n<p>Spider\u3068Engine\u3068\u306e\u9593\u306e\u51e6\u7406\u3092\u62e1\u5f35\u3059\u308b\u305f\u3081\u306b\u4f7f\u3044\u307e\u3059\u3002<\/p>\n<h4>Item<\/h4>\n<p>Spider\u3067\u306f\u53d6\u5f97\u3057\u305f\u30c7\u30fc\u30bf\u3092\u3053\u308c\u306b\u683c\u7d0d\u3057\u3066Engine\u3084Item pipeline\u3092\u901a\u3058\u3066\u9001\u308b\u3053\u3068\u3082\u53ef\u80fd\u3067\u3059\u3002<\/p>\n<h3>Engine<\/h3>\n<p>\u5168\u3066\u306e\u69cb\u6210\u8981\u7d20\u3092\u5236\u5fa1\u3057\u3066\u4e00\u8cab\u6027\u3092\u4fdd\u3064\u3088\u3046\u306b\u30b3\u30fc\u30c7\u30a3\u30cd\u30fc\u30c8\u3057\u3066\u304f\u308c\u307e\u3059\u3002<\/p>\n<h3>Scheduler<\/h3>\n<p>\u30ea\u30af\u30a8\u30b9\u30c8\u51e6\u7406\u3092\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u7ba1\u7406\u3057\u307e\u3059\u3002\u5177\u4f53\u7684\u306b\u306f\u30ea\u30af\u30a8\u30b9\u30c8\u306e\u9806\u756a\u3092\u30ad\u30e5\u30fc\u3068\u3044\u3046\u5f62\u3067\u4fdd\u5b58\u3057\u307e\u3059\u3002<\/p>\n<h3>Downloader<\/h3>\n<p>\u30ea\u30af\u30a8\u30b9\u30c8\u3067\u6307\u5b9a\u3055\u308c\u305fURL\u306eWeb\u30da\u30fc\u30b8\u3092\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9\u3057\u307e\u3059\u3002<\/p>\n<h4>Middleware<\/h4>\n<p>Downloader\u3068Engine\u3068\u306e\u9593\u306e\u51e6\u7406\u3092\u62e1\u5f35\u3059\u308b\u305f\u3081\u306b\u4f7f\u3044\u307e\u3059\u3002<\/p>\n<h3>Item Pipeline<\/h3>\n<p>\u62bd\u51fa\u3057\u305f\u30c7\u30fc\u30bf\u51e6\u7406\u306b\u95a2\u4fc2\u3057\u307e\u3059\u3002\u5177\u4f53\u7684\u306b\u306f\u30c7\u30fc\u30bf\u30af\u30ec\u30f3\u30b8\u30f3\u30b0\uff08\u30c7\u30fc\u30bf\u6574\u5f62\uff09\u3001\u91cd\u8907\u524a\u9664\u3001\u30c7\u30fc\u30bf\u30c1\u30a7\u30c3\u30af\u306a\u3069\u3092\u3057\u3066\u304f\u308c\u307e\u3059\u3002\u307e\u305f\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3078\u4fdd\u5b58\u3057\u305f\u3044\u5834\u5408\u306a\u3069\u3082\u3053\u306eItem Pipeline\u3092\u4f7f\u3063\u3066\u884c\u3044\u307e\u3059\u3002\u307e\u305f\u3001\u53d6\u5f97\u3057\u305f\u30c7\u30fc\u30bf(Item)\u3068\u53d6\u5f97\u9806\u306a\u3069\u304c\u683c\u7d0d\u3055\u308c\u3066\u3044\u307e\u3059\u3002<\/p>\n<p>\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3078\u4fdd\u5b58\u3057\u305f\u3044\u5834\u5408\u306f\u63a5\u7d9a\u65b9\u6cd5\u3084SQL\u304c\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u306b\u3088\u3063\u3066\u7570\u306a\u308b\u306e\u3067\u5fc5\u8981\u306b\u5fdc\u3058\u3066Item Pipeline\u306b\u51e6\u7406\u3092\u8a18\u8ff0\u3057\u3066\u3044\u304f\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n<h3>Feed Exporter<\/h3>\n<p>\u62bd\u51fa\u3057\u305f\u30c7\u30fc\u30bf\u3092CSV\u3084JSON\u306a\u3069\u306e\u30d5\u30a1\u30a4\u30eb\u306b\u51fa\u529b\u3057\u307e\u3059\u3002\u95a2\u5fc3\u4e8b\u3068\u3057\u3066\u306f\u3042\u304f\u307e\u3067\u30d5\u30a1\u30a4\u30eb\u51fa\u529b\u306b\u95a2\u3059\u308b\u3053\u3068\u3060\u3051\u3067\u3001\u30c7\u30fc\u30bf\u30d9\u30fc\u30b9\u3078\u306e\u4fdd\u5b58\u306b\u95a2\u3057\u3066\u306f\u5168\u304f\u95a2\u4e0e\u3057\u307e\u305b\u3093\u3002<\/p>\n<h2>\u51e6\u7406\u306e\u6d41\u308c<\/h2>\n<p>\u4e0a\u8a18\u306fScrapy\u306e\u69cb\u6210\u3092\u3054\u7d39\u4ecb\u3057\u307e\u3057\u305f\u304c\u5b9f\u969b\u306b\u3069\u306e\u3088\u3046\u306a\u6d41\u308c\u3067\u51e6\u7406\u304c\u884c\u308f\u308c\u3066\u3044\u308b\u304b\u3054\u7d39\u4ecb\u3057\u307e\u3059\u3002<\/p>\n<ol>\n<li>startURL\u306b\u6307\u5b9a\u3055\u308c\u305fURL\u306b\u5bfe\u3057\u3066spider\u306fEngine\u306b\u30ea\u30af\u30a8\u30b9\u30c8\u3092\u9001\u308a\u307e\u3059\u3002<\/li>\n<li>Engine\u306f\u30ea\u30af\u30a8\u30b9\u30c8\u3092Scheduler\u306b\u8ee2\u9001\u3057\u307e\u3059\u3002<\/li>\n<li>Scheduler\u304c\u30ea\u30af\u30a8\u30b9\u30c8\u3092\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u767b\u9332\u3057\u3001\u30b9\u30b1\u30b8\u30e5\u30fc\u30eb\u901a\u308a\u306b\u306a\u3063\u305f\u3089\u518d\u5ea6Engine\u306b\u30ea\u30af\u30a8\u30b9\u30c8\u304c\u9001\u3089\u308c\u307e\u3059\u3002<\/li>\n<li>Engine\u306fDownload Middleware\u3092\u7528\u3044\u3066Downloader\u306b\u30ea\u30af\u30a8\u30b9\u30c8\u3092\u9001\u308a\u307e\u3059\u3002<\/li>\n<li>Downloader\u306f\u30a4\u30f3\u30bf\u30fc\u30cd\u30c3\u30c8\u304b\u3089HTML\u306a\u3069\u306eWeb\u30da\u30fc\u30b8\u3092\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9\u3057\u307e\u3059\u3002<\/li>\n<li>\u30c0\u30a6\u30f3\u30ed\u30fc\u30c9\u3057\u305f\u30ec\u30b9\u30dd\u30f3\u30b9\u306fEngine\u306b\u9001\u4ed8\u3055\u308c\u307e\u3059\u3002<\/li>\n<li>Engine\u306fSpider Middleware\u3092\u7528\u3044\u3066Spider\u306b\u30ec\u30b9\u30dd\u30f3\u30b9\u7d50\u679c\u3092\u8fd4\u3057\u307e\u3059\u3002<\/li>\n<li>Spider\u3067\u306fXPath\u306a\u3069\u3092\u7528\u3044\u3066\u30ec\u30b9\u30dd\u30f3\u30b9\u304b\u3089\u30c7\u30fc\u30bf\u3092\u62bd\u51fa\u3059\u308b\u304b\u3001\u30ea\u30f3\u30af\u306a\u3069\u3092\u53d6\u5f97\u3057\u3066\u518d\u5ea6\u30ea\u30f3\u30af\u3092\u30ea\u30af\u30a8\u30b9\u30c8\u3057\u305f\u308a\u3057\u307e\u3059\u3002<\/li>\n<li>\u6700\u7d42\u7684\u306bSpider\u306f\u30ec\u30b9\u30dd\u30f3\u30b9\u7d50\u679c\u3092Python\u306e\u8f9e\u66f8\u306b\u683c\u7d0d\u3057\u3066\u3001Feed Exporter\u306a\u3069\u3067\u30d5\u30a1\u30a4\u30eb\u51fa\u529b\u3057\u307e\u3059\u3002<\/li>\n<\/ol>\n<p>\u4e0a\u8a18\u306f\u30011\u30ea\u30af\u30a8\u30b9\u30c8\u3092\u51e6\u7406\u3059\u308b\u6d41\u308c\u306b\u306a\u308a\u307e\u3059\u304c\u3001\u5b9f\u969b\u306fScrapy\u4e0a\u3067\u591a\u6570\u306e\u30ea\u30af\u30a8\u30b9\u30c8\u304c\u540c\u6642\u306b\u9ad8\u901f\u306b\u634c\u304b\u308c\u3066\u308b\u3088\u3046\u306b\u306a\u3063\u3066\u3044\u307e\u3059\u3002\u307e\u305a\u306f1\u30ea\u30af\u30a8\u30b9\u30c8\u306e\u6d41\u308c\u3092\u6291\u3048\u307e\u3057\u3087\u3046\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"\u69cb\u6210 Spider Scrapy\u3067\u306f\u4e3b\u306b\u30b3\u30fc\u30c9\u306f\u3053\u306e\u30af\u30e9\u30b9\u306b\u8a18\u8ff0\u3092\u3057\u3066\u3044\u304d\u307e\u3059\u3002\u3053\u3053\u306b\u8a18\u8ff0\u3059\u308c\u3070Scrapy\u304c\u3046\u307e\u3044\u3053\u3068\u4ed6\u306e\u4f5c\u696d\u3092\u3084\u3063\u3066\u304f\u308c\u308b\u3088\u3046\u306b\u306a\u308a\u307e\u3059\u3002 \u958b\u767a\u8005\u306fSpider\u3067\u4ee5\u4e0b\u306e\u7269\u3092\u6307\u5b9a\u3057\u307e\u3059\u3002 \u6700\u521d\u306b\u30a2\u30af\u30bb\u30b9 [&hellip;]","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[47],"tags":[],"_links":{"self":[{"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=\/wp\/v2\/posts\/20945"}],"collection":[{"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=20945"}],"version-history":[{"count":11,"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=\/wp\/v2\/posts\/20945\/revisions"}],"predecessor-version":[{"id":21113,"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=\/wp\/v2\/posts\/20945\/revisions\/21113"}],"wp:attachment":[{"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=20945"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=20945"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.code-magagine.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=20945"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}