Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 1384
Warning: preg_match_all(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 700
Warning: Invalid argument supplied for foreach() in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 707
Warning: preg_match_all(): Compilation failed: invalid range in character class at offset 4 in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 700
Warning: Invalid argument supplied for foreach() in /home/neginium/www/amimilab/wp/wp-content/plugins/lightbox-plus/classes/shd.class.php on line 707
PHPでWebページの内容を解析するときにjQueryみたいなセレクタ指定で解析できたら良いのにと思ったことはありませんか？
そこで今回はSymfonyと呼ばれるWeb用フレームワークに含まれるDOMCrawlerコンポーネントでWebページ解析する方法をご紹介します。
Contents
DOMCrawlerのインストール
まずはDOMCrawlerの前にComposerと呼ばれる依存管理ツールを導入します。
Composer.pharファイルをダウンロードするのみなのですぐに終わります。
※事前にPHP 5.5.9 以上が導入済みであり、パスが通っていることを前提とします。
今回はWindows機にインストールします。
作業ディレクトリは「C:\scraping_test」とします。
※Linux機でもほぼ同様の手順で導入可能です。
１．Composer.pharをダウンロード
Composerのダウンロードページへ行き、Manual Downloadの部分までスクロールします。
その後「Latest Snapshot」リンクをクリックして、Composer.pharをダウンロードします。
Composer.pharダウンロードページ
２．依存設定ファイル設定
手順１でダウンロードしたComposer.pharと同じディレクトリに、「composer.json」というファイルを作成して、以下の内容を記載・保存します。
{
"require": {
"symfony/dom-crawler": "3.2.*",
"symfony/css-selector": "3.2.*"
}
}
３．コマンドプロンプトで作業ディレクトリへ移動後、以下のコマンドを入力します。
php composer.phar install
以下のように表示されたら、インストール完了です。
作業用ディレクトリに「vendor」ディレクトリが作成されていることを確認します。
※もし、以下のようなエラーが出た場合は、PHPのSSLが無効となっている可能性があります。
php.iniを開き「;extension=php_openssl.dll」の行を探して、先頭の「;」を削除して上書き保存後、コマンドを再実行してください。
４．DOM Crawlerの動作を確認
以下のソースコードをtest.phpに保存して、動作を確認します。
スクリプトの処理は、ID:grp1のdiv内にあるClass:itemのspan要素を選択後、span要素内のテキストを表示するというものです。
<?php
require_once("./vendor/autoload.php");
use Symfony\Component\DomCrawler\Crawler;
$test_html =<<<HTML_END
<html>
<body>
<div id="grp1">
<span class="item">ITEM1-1</span>
<span class="item">ITEM1-2</span>
</div>
<div id="grp2">
<span class="item">ITEM2-1</span>
</div>
</body>
</html>
HTML_END;
$root = new Crawler();
//HTML内容はUTF-8エンコーディングとして解析
$root->addHTMLContent($test_html, "UTF-8");
foreach($root->filter("div#grp1 > span.item") as $item){
//foreachの戻り値はDOMElementなのでCrawlerに変換
$obj = new Crawler($item);
echo "TEXT: " . $obj->text() . "\n";
}
?>
実行すると、以下のような内容が表示され、WEBスクレイピングが行えたことが確認できます。
DomCrawlerの使い方
１．DomCrawlerオブジェクト・解析
以下のようにオブジェクトを作成し、addHTMLContentメソッドを実行することで、HTMLが解析されます。UTF-8の部分には、HTMLの文字コードをセットします。
$root = new Crawler(); $root->addHTMLContent($html_cont, "UTF-8");
２．filter実行
HTML解析後、filterメソッドを呼び出すことで、HTML要素を絞り込むことができます。
filterの構文はjQueryライクに指定できるので、jQueryを触ったことがあれば簡単にフィルターをかけることができます。
//フィルターの一例、他にもjQueryで使えるフィルタが使用できます
$result = $root->filter("#id"); //指定したidを持った要素を抜き出し
$result = $root->filter(".class"); //指定したclassを持った要素を抜き出し
３．各要素抜き出し
filterメソッドの戻り値はforeachで１つずつ取り出すことができます。
※取り出したデータの型はDOMElement型であるため、Crawlerクラスのメソッドを使えるよう、Crawlerクラスに渡してインスタンス化しています。
foreach($result as $element){
$obj = new Crawler($element);
echo $obj->text(); //要素内のテキストを表示
}活用例
Webスクレイピング活用一例を簡単にご紹介したいと思います。
ブログの最新記事を取ってくる
筆者のサイト(https://amimilab.com/)からタイトルとURLを抜き出すスクリプトを作成してみます。
<?php
require_once("./vendor/autoload.php");
use Symfony\Component\DomCrawler\Crawler;
$html_cont = file_get_contents("https://amimilab.com");
$root = new Crawler();
//HTML内容はUTF-8エンコーディングとして解析
$root->addHTMLContent($html_cont, "UTF-8");
foreach($root->filter("a.entry-title") as $item){
//foreachの戻り値はDOMElementなのでCrawlerに変換
$obj = new Crawler($item);
//コマンドプロンプトに表示するのでSJISに変換
echo toSJIS($obj->text() . "\n");
echo toSJIS(" LINK: " . $obj->attr("href") . "\n");
echo "\n";
}
//UTF8からSJISに変換する関数
function toSJIS($str){
return mb_convert_encoding($str, "SJIS-win", "UTF-8");
}
?>
上記スクリプトを実行すると、以下のようにコマンドプロンプト上に最新記事のタイトルとＵＲＬ一覧が表示されます。
まとめ
DOMCrawlerを使用すれば、PHPでjQueryライクに要素を抜き出すことができるためオススメです。
他にもPHPでDOM解析するものに「PHP Simple HTML DOM Parser」がありますが、こちらは更新日時が古いことや、HTMLによって正常に要素が抜き出せないことがあるため、DOMCrawlerをおススメしました。
この情報がお役に立てれば何よりです。