帝国后台导入新闻

作者: admin 分类: cms 发布时间: 2017-06-30 09:06

将采集到的新闻导入到数据库中

<?php 

include_once ('MysqlDB.class.php');
include_once("getSource.php");
$db = MysqlDB::getInstance('127.0.0.1', 'multilang.sbmchina.com', 'root','');

//$data = $db->fetch("phome_ecms_news_index"," WHERE id =655");

//$data=$query->fetchAll();
//print_r($data);


$urls=getUrls("urls/ar_url.log");
//
$filename=298;
foreach ($urls as $url) {

	$source=getSource($url);
	$title = $source['title'];
	$time = $source['time'];
	$content = $source['content'];
	//产品数据表中字段
	$newsfield = array(
	    "classid" => 56,
	    "ttid" => 0,
	    "onclick" => 0,
	    "plnum" => 0,
	    "totaldown" => 0,
	    "newspath" => "",
	    "filename" => $filename,
	    "userid" => 1,
	    "username" =>"超管",
	    "firsttitle" => 0,
	    "isgood" => 0,
	    "ispic" => 0,
	    "istop" => 0,
	    "isqf" => 0,
	    "ismember" => 0,
	    "isurl" => 0,
	    "truetime" => $time,
	    "lastdotime" => $time,
	    "havehtml" => 1,
	    "groupid" => 0,
	    "userfen" => 0,
	    "titlefont" => "",
	    "titleurl" => "http://fr.sbmchina.com/media/news/{$filename}.html",
	    "stb" => 1,
	    "fstb" => 1,
	    "restb" => 1,
	    "keyboard" => "",
	    "title" => $title,
	    "newstime" => $time,
	    "titlepic" => "",
	    "ftitle" => "",
	    "smalltext" => $title,
	    "diggtop" => 0,
	    "relatedmachine" => "",
	    "relatedinfo" => "",
	    "classname"=>"",
	    "tags"=>""
	);
	$data = $db->insert("phome_ecms_news", $newsfield);
	echo "insert phome_ecms_news success".PHP_EOL;
	$re = $db->query("select * from phome_ecms_news where title='{$title}'");
	$result=$re->fetch();
	$id=$result['id'];
	$classid=$result['classid'];
	//产品数据分表中字段
	$proDataField = array(
	    "id" => $id,
	    "classid" => $classid,
	    "keyid" =>"",
	    "dokey" => 1,
	    "newstempid" => 0,
	    "closepl" => 0,
	    "haveaddfen" => 0,
	    "infotags" =>"" ,
	    "writer" =>"",
	    "befrom"=>"",
	    "newstext"=>$content

	    
	);

	$data = $db->insert("phome_ecms_news_data_1", $proDataField);
	echo "insert phome_ecms_news_data_1 success".PHP_EOL;
	//产品索引表中字段
	$proIndex = array(
	    "id" => $id,
	    "classid" => $classid,
	    "checked" => 1,
	    "newstime" => $time,
	    "truetime" => $time,
	    "lastdotime" => $time,
	    "havehtml" => 1
	);
	//print_r($proIndex);
	$data = $db->insert("phome_ecms_news_index", $proIndex);
	echo 'insert into phome_ecms_news_index success'.PHP_EOL;
	$filename++;
}
$db->close();

 ?>

根据url抓取内容

<?php 

@header("Content-Type: text/html; charset=UTF-8");
//不限执行时间
set_time_limit(0); 
error_reporting(0);
//从日志文件中获取url
function getUrls($fileName){
	$lineArray=array();
	$fp = fopen($fileName, "rb") or die("Unable to open file ".$fileName);
	
	while(! feof($fp))
	{
		$line=fgets($fp);
		$line=trim($line);
		$lineArr[]=$line;

	}

	fclose($fp);
	return $lineArr;

}


//根据url获取内容
function getSource($url){
	$title="";
	$time="";
	$content="";
	

	require_once('phpQuery/phpQuery.php');
	$html=phpQuery::newDocumentFile($url);         
	//获取描述内容
	$title=$html->find('.newstitle')->html();
	$time=$html->find('.newstime')->html();
	$time=strtotime($time);
	$content=$html->find('.newscontent')->html();
	

	$search = array(        
	                    "~<span(.*?)>~is",
	                     "~<p (.*?)>~is"
	                                      
	                                );
	                                
	$replace = array(        
	                    "<span>",
	                    "<p>"
	                                       
	                                );                        
	//去除样式,去除后效果不是太好不用处理
	//$content = preg_replace($search,$replace,$content);
	$content=mb_convert_encoding($content, 'utf-8',mb_detect_encoding($content));    
	$title=mb_convert_encoding($title, 'utf-8',mb_detect_encoding($title));
	//$title=htmlspecialchars($title);
	$title= addslashes($title);
	//$title=mysql_escape_string ($title);
	$content=addslashes($content);
	//$content=mysql_escape_string ($content);
	//$title=clear($title);
	return array('title'=>$title,'time'=>$time,'content'=>$content);

}


function clear($kw)
{ 
    $kw= str_replace(array("~" ,"!" ,"@" ,"#" ,"$" ,"%" ,"^" ,"+","&" ,"*" ,"," ,"." ,"?" ,";",":" ,'\'','"' ,"[" ,"]" ,"{" ,"}" ,"!" ,"¥" ,"……" ,"…" ,"、" ,"," ,"。" ,"?" ,";" ,":","'","“" ,"”" ,"'" ,"【" ,"】" ,"~" ,"!" ,"@" ,"#" ,"$" ,"%" ,"^" ,"&" ,"*" ,"," ,"." ,"<" ,">" ,";" ,":","'",""" ,"[" ,"]" ,"{" ,"}","/" ,"\" ,"(" ,")" ,"(" ,")","《","》", '$','¿','×'),'', $kw ); 
		
    $kw= str_replace( array("  ","-","_","\\","/","|"),' ', $kw ); 
	
    //$kw= str_replace(array("á","í","é","ó","ú","ñ","Á","Í","É","Ó","Ú","Ñ","ç","ã","à","â","ê","ô","õ","ü"),array("a","i","e","o","u","n","a","i","e","o","u","n","c","a","a","a","e","o","o","u"),$kw);
	
    //$kw= str_replace(array("а","б","в","г","д","е","ё","ж","з","и","й","к","л","м","н","о","п","р","с","т","у","ф","х","ц","ч","ш","я","ю","щ","щ","э","ъ","ь","А","Б","В","Г","Д","Е","Ё","Э","Ж","З","И","Й","К","Л","М","Н","О","П","Р","С","Т","У","Ф","Х","Ц","Ч","Ш","Щ","Ы","Ю","Я","ы"),array("a","b","v","g","d","e","e","zh","z","i","j","k","l","m","n","o","p","r","s","t","u","f","x","c","ch","s","ya","yu","sch","y","e","","","A","B","V","G","D","E","E","E","J","Z","I","I","K","L","M","N","O","P","R","S","T","U","F","H","C","CH","SH","SH","Y","YU","YA","s"),$kw);
    
	//$kw = strtolower(strip_tags(trim($kw)));    
	
	//$kw = explode(' ',$kw);
	
	//$kw = implode('-',array_filter($kw));
	
	return $kw;
} 


// foreach ($urls as $url) {
// 	$content=getSource($url);
// 	echo $url.PHP_EOL;
// 	print_r($content);
// }

//$content=getSource("http://fr.leevii.com/news/204.html");
//print_r($content);

 ?>

数据库操作函数

<?php
class MysqlDB
{
    protected static $_instance = null;
    protected $dsn;
    protected $pdo;
    

    private function __construct($dbHost, $dbName,$dbUser, $dbPasswd, $dbCharset)
    {
        try {
            $this->dsn = 'mysql:host='.$dbHost.';dbname='.$dbName;
            $this->pdo = new PDO($this->dsn, $dbUser, $dbPasswd);
            $this->pdo->exec('SET character_set_connection='.$dbCharset.', character_set_results='.$dbCharset.', character_set_client=binary');
        } catch (PDOException $e) {
            echo '数据库连接失败:'.$e->getMessage();
            exit;
        }
    }
    
    //防止PDO类被克隆
    private function __clone() {}
    
   
    public static function getInstance($dbHost='127.0.0.1', $dbName='test',$dbUser='root', $dbPasswd='', $dbCharset='utf8')
    {
        if (self::$_instance === null) {
            self::$_instance = new self($dbHost, $dbName,$dbUser, $dbPasswd, $dbCharset);
        }
        return self::$_instance;
    }
    
  
    public function fetch($table, $where="",$queryMode = 'All')
    {
        if(!empty($where)){
            $strSql='select * from '.$table.' '.$where;
        }else{
            $strSql='select * from '.$table;
        }
        
        $recordset = $this->pdo->query( $strSql);
       
        if ($recordset) {
            $recordset->setFetchMode(PDO::FETCH_ASSOC);
            if ($queryMode == 'All') {
                $result = $recordset->fetchAll();
            } elseif ($queryMode == 'Row') {
                $result = $recordset->fetch();
            }
        } else {
            $result = null;
        }
        return $result;
    }
    
    
    public function update($table, $arrayDataValue, $where = '')
    {
       
        if ($where) {
            $strSql = '';
            foreach ($arrayDataValue as $key => $value) {
                $strSql .= ", '$key'='$value'";
            }
            $strSql = substr($strSql, 1);
            $strSql = "UPDATE $table SET $strSql WHERE $where";
        } else {
            $strSql = "REPLACE INTO $table (".implode(',', array_keys($arrayDataValue)).") VALUES ('".implode("','", $arrayDataValue)."')";
        }
       
        $result = $this->pdo->exec($strSql);
        
        return $result;
    }
    
  
    public function insert($table, $arrayDataValue)
    {
       
        $strSql = "INSERT INTO $table (".implode(',', array_keys($arrayDataValue)).") VALUES ('".implode("','", $arrayDataValue)."')";
      
        $result = $this->pdo->exec($strSql);
        return $result;
    }
    

    public function delete($table, $where = '')
    {
        if ($where == '') {
            echo "'WHERE' is Null";
        } else {
            $strSql = "DELETE FROM $table WHERE $where";
           
            $result = $this->pdo->exec($strSql);
            
            return $result;
        }
    }
   
    public function exec($strSql)
    {
        $result = $this->pdo->exec($strSql);
        return $result;
    }
    public function query($strSql)
    {
        $result = $this->pdo->query($strSql);
        return $result;
    }

    public function close()
    {
        $this->pdo = null;
    }
}



?>


 

 

如果觉得我的文章对您有用,请随意打赏。您的支持将鼓励我继续创作!

发表评论

电子邮件地址不会被公开。 必填项已用*标注