原本一筆筆資料insert,52萬筆花了3小時,
改用Mysql的LOAD DATA從產生csv檔到Load Data至新table只花了5分鐘不到,
範例如下,請參考~
public boolean runByAll(String userName, String tableName) {
boolean bSuccess = false;
String filePath = ...; //取得暫存檔路徑
BufferedWriter writer = null;
String fileNameWithPath = filePath + File.separator + fileName;
File file = new File(fileNameWithPath);
if (file != null && file.exists() == true) {
boolean bDeleteOK = file.delete();
if (bDeleteOK == false) {
logger.warn("刪除舊資料- " + fileNameWithPath + " 失敗.");
}
}
//1.將來源資料寫入暫存檔案 : csv格式
ServiceEntry serviceEntry = new ServiceEntry();
Session session = null;
int iTotalRow = 0;
Date startDate = new Date();
try {
writer = new BufferedWriter(new FileWriter(fileNameWithPath));
session = serviceEntry.getSessionforTrf();
ScrollableResults table1Cursor
= session.createQuery("FROM " + tableName)
.setReadOnly(true)
.setCacheable(false)
.setFetchSize(1000)
.scroll(ScrollMode.FORWARD_ONLY);
while (table1Cursor.next()) {
Table1 table1 = (Table1) table1Cursor.get(0);
iTotalRow++;
Table1PK table1PK = table1.getTable1PK();
String bufferStr
= SystemDateTime.getFmtDate(table1PK.getTxDate(),
"yyyy-MM-dd HH:mm:ss") + "," + table1PK.getStockId() + "," +
table1.getStockName() + "," + table1.getToUp() + "," +
table1.getToDown() + "," + table1.getLastPrice() + "," +
table1.getuB() + "," + table1.getlB() + "," +
table1.getAvgPrice() + "," +
SystemDateTime.getFmtDate(startDate, "yyyy-MM-dd HH:mm:ss") +
"," + userName + "\n";
writer.write(bufferStr);
if (iTotalRow % 1000 == 0) {
logger.info("count now : " + iTotalRow);
session.flush();
session.clear();
writer.flush();
}
}
session.close();
bSuccess = true;
logger.info("原始資料共:" + iTotalRow);
} catch (Exception e) {
logger.error(e);
} finally {
serviceEntry.closeSessionforTrf(session);
if (writer != null) {
try {
writer.close();
} catch (IOException e) {
logger.error(e);
}
}
}
if (bSuccess == false) {
return bSuccess;
}
//大量資料寫入另一Table
ServiceEntry2 serviceEntry2 = new ServiceEntry2();
Session session2 = null;
int dbRowCount = 0;
try {
//1) truncate table
session2 = serviceEntry2.getSessionforTrf();
session2.createSQLQuery("truncate table " + tableName).executeUpdate();
//2) Load Data from csv File
String sHql_load = "LOAD DATA LOCAL INFILE :filename " +
" INTO TABLE " + tableName +
" FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' " +
" (@var_TxDate, StockID, StockName, toUp, toDown, LastPrice, " +
" UB, LB, AvgPrice, @var_ModifyTime, ModifyEmp) " +
" SET TxDate = STR_TO_DATE(@var_TxDate,'%Y-%m-%d %H:%i:%S'), " +
" ModifyTime = STR_TO_DATE(@var_ModifyTime,'%Y-%m-%d %H:%i:%S');";
Query query_load = session2.createSQLQuery(sHql_load)
.setString("filename", fileNameWithPath);
dbRowCount = query_load.executeUpdate();
bSuccess = (dbRowCount == iTotalRow) ? true : false;
} catch (Exception e) {
logger.error(e);
bSuccess = false;
} finally {
serviceEntry.closeSessionforTrf(session2);
}
return bSuccess;
}
Ref.http://www.codedata.com.tw/database/mysql-tutorial-19-outfile-dump-infile-import/