perfection the data-import functions

This commit is contained in:
Li Jianxuan 2015-01-30 07:47:53 +00:00
parent b1d256b679
commit 7136dc98ad
6 changed files with 173 additions and 25 deletions

View File

@ -4265,8 +4265,8 @@ class Admin_DataController extends Zend_Controller_Action
return true; return true;
} }
//检查能否写入数据表 //检查能否写入数据表 和 导入数据
if($ac == "checktables"){ if($ac == "checktables" || $ac == "import"){
$this->_helper->layout->disableLayout(); $this->_helper->layout->disableLayout();
$this->_helper->viewRenderer->setNoRender(); $this->_helper->viewRenderer->setNoRender();
@ -4278,9 +4278,6 @@ class Admin_DataController extends Zend_Controller_Action
return true; return true;
} }
$tableControl = new Visual\DataTableControl;
$fields = $tableControl->readFields($schema . "." .$table);
$fileHandle = new Dataimport\File; $fileHandle = new Dataimport\File;
$realFile = $fileHandle->getRealName($file); $realFile = $fileHandle->getRealName($file);
@ -4288,11 +4285,23 @@ class Admin_DataController extends Zend_Controller_Action
/** @var \Westdc\Dataimport\Processing\Csv $processing */ /** @var \Westdc\Dataimport\Processing\Csv $processing */
$processing->init($realFile); $processing->init($realFile);
$status = $processing->checkTableField($fields); if($ac == "checktables")
{
$tableControl = new Visual\DataTableControl;
$fields = $tableControl->readFields($schema . "." .$table);
//$this->_helper->json($status); $status = $processing->checkTableField($fields);
$this->_helper->json($status);
return true;
}
elseif($ac == "import"){
$status = $processing->import($schema . "." .$table);
$this->_helper->json($status);
return true;
}
return true; return true;
} }
return true; return true;

View File

@ -41,6 +41,8 @@ $this->headTitle()->setSeparator(' - ');
<input type="hidden" name="schema" value=""> <input type="hidden" name="schema" value="">
<input type="hidden" name="table" value="" id="input-table"> <input type="hidden" name="table" value="" id="input-table">
<button type="button" id="ctl-next" class="btn btn-success disabled" disabled>下一步</button> <button type="button" id="ctl-next" class="btn btn-success disabled" disabled>下一步</button>
<button type="button" id="ctl-show-table" class="btn btn-info hidden" disabled>查看数据表结构</button>
<button type="button" id="ctl-import" class="btn btn-info" disabled="disabled" style="display: none;">导入数据表</button>
</form> </form>
</div> </div>
</div> </div>
@ -51,6 +53,45 @@ $this->headTitle()->setSeparator(' - ');
ctl.checkTable(); ctl.checkTable();
}); });
$("#ctl-show-table").click(function(){
});
$("#ctl-import").click(function(){
$.ajax({
'type':"POST",
'url':'/admin/data/dataimport/ac/import',
'data':$('#schema-form').serialize(),
'success':function(data){
if (data!=true)
{
if(!$.isEmptyObject(data))
$(".error-alerts").show();
$("#errors").html(data.join("<br />"));
return true;
}else{
$('#ctl-import').remove();
$("#errors").html("数据导入成功!页面将自动跳转到数据可视化管理页面");
setTimeout("top.location = '/admin/data/visual';",3000);
}
},
'timeout': 30000,
'beforeSend' : function(){
$('#error-loading').html("正在进行数据导入......");
$('#error-loading').show();
$('#ctl-import').attr('disabled','disabled');
},
'complete' : function(){
$('#error-loading').hide();
},
'error' : function(){
$('#ctl-import').removeAttr('disabled');
}
});
});
$('.ctl-schema-btn').click(function(){ $('.ctl-schema-btn').click(function(){
$('#schema-form input[name=schema]').val($(this).val()); $('#schema-form input[name=schema]').val($(this).val());
@ -124,13 +165,23 @@ $this->headTitle()->setSeparator(' - ');
'url':'/admin/data/dataimport/ac/checktables', 'url':'/admin/data/dataimport/ac/checktables',
'data':$('#schema-form').serialize(), 'data':$('#schema-form').serialize(),
'success':function(data){ 'success':function(data){
if (data!=null) if (data!=true)
{ {
if(!$.isEmptyObject(data))
$(".error-alerts").show();
$("#errors").html(data.join("<br />"));
return true; return true;
}else{ }else{
alert('处理中出现错误'); $(".error-alerts").show();
$('#errors').html('数据准备完成,可以进行导入操作');
$('.ctl-schema-btn').attr("disabled","disabled");
$('.ctl-table-name').attr("disabled","disabled");
$('#ctl-next').hide();
$('#ctl-show-table').hide();
$('#ctl-import').show();
$('#ctl-import').removeAttr("disabled");
} }
}, },
'timeout': 30000, 'timeout': 30000,

View File

@ -5,8 +5,6 @@ class PDO extends \PDO
{ {
private $debug = 0; //调试模式 private $debug = 0; //调试模式
private $config_local_path = "config/autoload/local.php";
public $db_cfg; public $db_cfg;
function __construct($DSN = NULL) function __construct($DSN = NULL)
@ -14,7 +12,7 @@ class PDO extends \PDO
if(empty($DSN)) if(empty($DSN))
{ {
$config_local = new Zend_Config(include $this->config_local_path); $config_local = \Zend_Registry::get('config');
$dsn = "pgsql:host={$config_local->db->hostname};" $dsn = "pgsql:host={$config_local->db->hostname};"
."port=5432;" ."port=5432;"
."dbname={$config_local->db->database};" ."dbname={$config_local->db->database};"

View File

@ -0,0 +1,40 @@
<?php
/**
* Created by PhpStorm.
* User: liujin834
* Date: 15-1-27
* Time: 下午2:37
*/
namespace Westdc\Dataimport;
abstract class AbstractProcessing {
/**
* 获取判断数据类型的函数
* @param $pgsql_data_type
* @param $data
* @return null|string
*/
public function checkFieldsDataType($pgsql_data_type,$data){
switch ($pgsql_data_type){
case "integer":
case "bigint":
case "int2vector":
case "real":
case "double precision":
return is_int($data) || is_float($data);
case "date":
case "char":
case "character varying":
return is_string($data);
default:
return FALSE;
}
}
}

View File

@ -44,7 +44,7 @@ class File{
} }
$ext = $this->getFileTextExt($file['name']); $ext = $this->getFileTextExt($file['name']);
$filename = $this->gen_uuid().".".$ext; $filename = $file['name']."_".$this->gen_uuid().".".$ext;
$new_filepath = $this->getSavePath().$filename; $new_filepath = $this->getSavePath().$filename;

View File

@ -8,9 +8,11 @@
namespace Westdc\Dataimport\Processing; namespace Westdc\Dataimport\Processing;
use Westdc\Dataimport\ProcessingInterface; use Westdc\Dataimport\ProcessingInterface;
use Westdc\Dataimport\AbstractProcessing;
use Goodby\CSV as CSVOBJ; use Goodby\CSV as CSVOBJ;
use Helpers\PDO;
class Csv implements ProcessingInterface{ class Csv extends AbstractProcessing implements ProcessingInterface{
private $source_file; private $source_file;
private $csv_object; private $csv_object;
@ -20,6 +22,8 @@ class Csv implements ProcessingInterface{
const ERROR_INDEX_TYPE_DIFFERENT = "值的类型于上一行不同"; const ERROR_INDEX_TYPE_DIFFERENT = "值的类型于上一行不同";
const ERROR_EMPTY_FILE = "文件是空的"; const ERROR_EMPTY_FILE = "文件是空的";
const ERROR_LINE_TITLE = "行:"; const ERROR_LINE_TITLE = "行:";
const ERROR_COLUMN_TITLE = "列:";
const ERROR_DATA_TYPE_NOT_MATCHED = "数据与数据库中的类型不匹配,不能进行导入";
/** /**
* 初始化 * 初始化
@ -31,6 +35,10 @@ class Csv implements ProcessingInterface{
$interpreter = new CSVOBJ\Import\Standard\Interpreter; $interpreter = new CSVOBJ\Import\Standard\Interpreter;
$interpreter->addObserver(function($columns){ $interpreter->addObserver(function($columns){
foreach($columns as $k=>$v){
if(is_numeric($v))
$columns[$k] = $v+0;
}
$this->csv_line_data[] = $columns; $this->csv_line_data[] = $columns;
}); });
$lexer = new CSVOBJ\Import\Standard\Lexer(new CSVOBJ\Import\Standard\LexerConfig); $lexer = new CSVOBJ\Import\Standard\Lexer(new CSVOBJ\Import\Standard\LexerConfig);
@ -61,7 +69,7 @@ class Csv implements ProcessingInterface{
$size/=1024; $size/=1024;
} }
return (floor($size*100)/100).$type[$i];//floor是取整函数为了防止出现一串的小数这里取了两位小数 return (floor($size*100)/100).$type[$i];//floor是取整函数为了防止出现一串的小数这里取了两位小数
}//getSize() }//getSize(){
/** /**
* 獲取文件類型 * 獲取文件類型
@ -101,9 +109,9 @@ class Csv implements ProcessingInterface{
} }
/** /**
* 檢查文件內容是否規則 * 檢查文件內容是否規則
* @return array|bool * @return array|bool
*/ */
public function checkRegularity() public function checkRegularity()
{ {
$lines = $this->getLines(); $lines = $this->getLines();
@ -148,22 +156,64 @@ class Csv implements ProcessingInterface{
{ {
$error_stack = []; $error_stack = [];
$frist_line = $this->getLines()[0]; $first_line = $this->getLines()[0];
var_dump($fields);
foreach($fields as $index => $field) foreach($fields as $index => $field)
{ {
if($field['data_type']){ $func = $this->checkFieldsDataType($field['data_type'],$first_line[$index]);
} if($func)
continue;
else
$error_stack[] = self::ERROR_COLUMN_TITLE . $index .":" . self::ERROR_DATA_TYPE_NOT_MATCHED;
} }
if(0) if(count($error_stack) < 1)
return true; return true;
else else
return $error_stack; return $error_stack;
} }
public function import($table)
{
$config = \Zend_Registry::get('config');
$db = new PDO([
'host' => $config->visual_db->hostname,
'port' => $config->visual_db->port,
'db' => $config->visual_db->database,
'user' => $config->visual_db->username,
'pwd' => $config->visual_db->password
]);
$db->setAttribute(\PDO::ATTR_PERSISTENT,true);
$error_stack = [];
try{
$db->setAttribute(\PDO::ATTR_ERRMODE , \PDO::ERRMODE_EXCEPTION);
$db->beginTransaction();
$place_holder=[];
foreach($this->getLines()[0] as $v)
{
$place_holder[] = "?";
}
$sth = $db->prepare("INSERT INTO $table VALUES (".join(",",$place_holder).")");
foreach($this->getLines() as $line)
{
$sth->execute($line);
}
$db->commit();
}catch(\Exception $e){
$db->rollBack();
$error_stack[] = $e->getMessage();
return $error_stack;
}
return true;
}
} }