百万行Excel数据全自动导入MYSQL之Go

执行文件与excel表格在同一个目录下

package main

import (
	"database/sql"
	"fmt"
	"strconv"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/mozillazg/go-pinyin"
	"github.com/xuri/excelize/v2"
)

//数据库配置
const (
	userName = "用户名"
	password = "密码"
	ip       = "数据库ip地址"
	port     = "端口"
	dbName   = "数据库名"
)

//Db数据库连接池
var DB *sql.DB

func InitDB() {
	//构建连接:"用户名:密码@tcp(IP:端口)/数据库?charset=utf8"
	path := strings.Join([]string{userName, ":", password, "@tcp(", ip, ":", port, ")/", dbName, "?charset=utf8"}, "")

	DB, _ = sql.Open("mysql", path)
	//设置数据库最大连接数
	DB.SetConnMaxLifetime(100)
	//设置上数据库最大闲置连接数
	DB.SetMaxIdleConns(10)
	//验证连接
	if err := DB.Ping(); err != nil {
		fmt.Println("opon database fail")
		return
	}
	fmt.Println("connnect success")
}

func exec_sql(insert_sql string, data []interface{}) (sql.Result, error) {
	tx, _ := DB.Begin()
	result, err := tx.Exec(insert_sql, data...)
	if err != nil {
		tx.Rollback().Error()
	} else {
		tx.Commit()
	}
	return result, err
}

func py(words string) string {
	words = strings.TrimSpace(words)
	list_words := pinyin.LazyConvert(words, nil)
	words = ""
	for i, v := range list_words {
		if i != 0 {
			words = strings.Join([]string{words, v}, "_")
		} else {
			words = strings.Join([]string{words, v}, "")
		}
	}
	return words
}

func ph(len_col int) string {
	placeHolders := "(?"
	for i := 0; i < len_col; i++ {
		placeHolders = strings.Join([]string{placeHolders, "?"}, ",")
	}
	placeHolders = strings.Join([]string{placeHolders, ")"}, "")
	return placeHolders
}

func main() {
	start_time_count := time.Now() //获取当前时间
	go InitDB()
  // 文件名
	file_name := "***.xlsx"
	file_name_slice := strings.Split(file_name, ".")
	table_name := file_name_slice[0]
	f, err := excelize.OpenFile(file_name)
	if err != nil {
		fmt.Println(err)
		return
	}
  // 工作薄名称默认Sheet1
	rows, err := f.Rows("Sheet1")
	go fmt.Printf("打开文件耗时 = %v\n", time.Since(start_time_count))
	data_excels := []interface{}{}
	value_strings := make([]string, 0)

	col_name_sql_creats := "CREATE TABLE IF NOT EXISTS " + table_name + "( "

	// 每次插入 行
	deal_c := 1000
	deal_c_now := 0
	placeHolders := ""
	data_counts := 0
	go fmt.Println("正在导入数据……")
	start_time_insert := time.Now()
	for rows.Next() {
		deal_c_now++
		data_counts++
		row, _ := rows.Columns()
		if data_counts == 1 {
			len_col := len(row)
			placeHolders = ph(len_col)
			fmt.Printf("有 %v 列\n", strconv.Itoa(len_col))
			for i, _ := range row {
				c := "c"
				c = strings.Join([]string{c, strconv.Itoa(i+1) + " VARCHAR(255), "}, "")
				col_name_sql_creats = strings.Join([]string{col_name_sql_creats, c}, " ")
			}
			col_name_sql_creats = strings.Join([]string{col_name_sql_creats, "id INT NOT NULL AUTO_INCREMENT, PRIMARY KEY (id) )"}, "")
			DB.Exec(col_name_sql_creats)
			fmt.Println(col_name_sql_creats)
		}
		for _, colCell := range row {
			data_excels = append(data_excels, colCell)
		}

		data_excels = append(data_excels, "")
		value_strings = append(value_strings, placeHolders)
		for deal_c_now > deal_c {
			insert_sql := fmt.Sprintf("INSERT INTO "+table_name+"  VALUES %v", strings.Join(value_strings, ","))
			exec_sql(insert_sql, data_excels)
			// go fmt.Printf("已导入%v条数据\n", deal_c)
			data_excels = []interface{}{}
			value_strings = make([]string, 0)
			deal_c_now = 0
		}
	}
	insert_sql := fmt.Sprintf("INSERT INTO ggxh VALUES %v", strings.Join(value_strings, ","))
	exec_sql(insert_sql, data_excels)
	fmt.Printf("最后一次导入%v条数据\n", deal_c_now)
	fmt.Printf("共计导入%v条数据\n", data_counts)
	fmt.Printf("insert into总时间 = %v\n", time.Since(start_time_insert))
	fmt.Printf("运行 总耗时 = %v\n", time.Since(start_time_count))
	defer DB.Close()
}