Attachment 653376 Details for Bug 880752 – catalog-parser.php

catalog-parser.php

catalog-parser.php (text/plain), 3.06 KB, created by Remi Collet on 2012-11-28 09:11:35 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Remi Collet

Created: 2012-11-28 09:11:35 UTC

Size: 3.06 KB

patch

obsolete

>#!/usr/bin/php
><?php
>include 'simple_html_dom.php';
>// Ð³Ð¾ÑÐ¾Ð´, ÐºÐ°ÑÐµÐ³Ð¾ÑÐ¸Ñ, Ð¿Ð¾Ð´ÐºÐ°ÑÐµÐ³Ð¾ÑÐ¸Ñ, ÐÐ´ÑÐµÑ, Ð¢ÐµÐ»ÐµÑÐ¾Ð½, Email , Ð¡Ð°Ð¹Ñ, ÐÑÐµÐ¼Ñ ÑÐ°Ð±Ð¾ÑÑ,
>// http://www.62.ua/info/citysites
>
>function cache_get_html($link) {
>	$tmp = 'cache/'.md5($link);
>	if (file_exists($tmp)) {
>		$str = file_get_contents($tmp);
>	} else {
>		$str = file_get_contents($link);
>		file_put_contents($tmp, $str);
>	}
>	return ($str ? str_get_html($str) : NULL);
>}
>
>function usedmem() {
>	return sprintf("\t(%.2f)\n", memory_get_usage()/1024/1024);
>}
>
>function processCity($cityName, $cityLink)
>{
>    echo "$cityName: $cityLink\n";
>
>    $html = cache_get_html($cityLink.'/catalog');
>
>    foreach($html->find('div.cat_menu a[data-id]') as $catLink) {
>        echo "    {$catLink->plaintext}\n";
>        $catId = $catLink->getAttribute('data-id');
>        foreach($html->find("#parent_$catId a[data-id]") as $subcatLink) {
>            echo "\t{$subcatLink->plaintext}: $cityLink{$subcatLink->href}".usedmem();
>            processCategory($cityName, trim($catLink->plaintext), trim($subcatLink->plaintext), $cityLink, $subcatLink->href);
>        }
>    }
>}
>
>function processCategory($cityName, $catName, $subcatName, $cityLink, $subcatLink)
>{
>	$pid = pcntl_fork();
>	if ($pid>0) {
>		pcntl_waitpid($pid, $status);
>	} else {
>		$csv = fopen('citysites.csv', 'a');
>		
>		$html = cache_get_html($cityLink.$subcatLink);
>		foreach($html->find('div.company_box h2 a') as $a) {
>		    echo "\t    {$a->href}".usedmem();
>		    parseCompany($csv, $cityName, $catName, $subcatName, $cityLink, $a->href);
>		}
>		fclose($csv);
>		exit(0);
>    }
>}
>
>function parseCompany($csv, $cityName, $catName, $subcatName, $cityLink, $companyLink)
>{
>    $company = array();
>    
>    $html = cache_get_html(substr($companyLink,0,14) == '/catalog/full/' ? $cityLink.$companyLink : $companyLink);
>    if ($html) foreach($html->find('div.info div.block_info div.info_box') as $ib) {
>        $company[trim($ib->childNodes(0)->plaintext)] = trim($ib->childNodes(1)->plaintext);
>    }
>
>    $line = array(
>        $cityName,
>        $catName,
>        $subcatName,
>        empty($company['ÐÐ´ÑÐµÑ']) ? '' : $company['ÐÐ´ÑÐµÑ'],
>        empty($company['Ð¢ÐµÐ»ÐµÑÐ¾Ð½']) ? '' : $company['Ð¢ÐµÐ»ÐµÑÐ¾Ð½'],
>        empty($company['Email Ð°Ð´ÑÐµÑ']) ? '' : $company['Email Ð°Ð´ÑÐµÑ'],
>        empty($company['Ð¡Ð°Ð¹Ñ']) ? '' : $company['Ð¡Ð°Ð¹Ñ'],
>        empty($company['ÐÑÐµÐ¼Ñ ÑÐ°Ð±Ð¾ÑÑ']) ? '' : $company['ÐÑÐµÐ¼Ñ ÑÐ°Ð±Ð¾ÑÑ'],
>    );
>    fputcsv($csv, $line, ';');
>}
>
>// truncate
>$csv = fopen('citysites.csv', 'w');
>fclose($csv);
>/*
>$citysites = file_get_html('http://www.62.ua/info/citysites');
>foreach($citysites->find('td.contact a') as $link) {
>    processCity($link->plaintext, $link->href);
>}
>*/
>processCity('ÐÐ´ÐµÑÑÐ°', 'http://www.048.ua');
>//processCategory('ÐÐ´ÐµÑÑÐ°', 'ÐÐ²ÑÐ¾', 'ÐÐ²Ð°ÑÐ¸Ð¹Ð½Ð¾Ðµ Ð²ÑÐºÑÑÑÐ¸Ðµ  Ð°Ð²ÑÐ¾Ð·Ð°Ð¼ÐºÐ¾Ð²', 'http://www.048.ua', '/catalog/23/580');
>//parseCompany('ÐÐ´ÐµÑÑÐ°', 'ÐÐ²ÑÐ¾', 'ÐÐ²Ð°ÑÐ¸Ð¹Ð½Ð¾Ðµ Ð²ÑÐºÑÑÑÐ¸Ðµ  Ð°Ð²ÑÐ¾Ð·Ð°Ð¼ÐºÐ¾Ð²', 'http://www.048.ua', 'http://autokluch.048.ua/');
>

Actions: View

Attachments on bug 880752: 652987 | 652988 | 652989 | 652990 | 652991 | 652992 | 652993 | 652994 | 652995 | 652996 | 652997 | 652998 | 653056 | 653376