intial commit
This commit is contained in:
388
internal/backup/mysql_to_parquet_orders.go
Normal file
388
internal/backup/mysql_to_parquet_orders.go
Normal file
@@ -0,0 +1,388 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/apache/arrow/go/v14/arrow"
|
||||
"github.com/apache/arrow/go/v14/arrow/array"
|
||||
"github.com/apache/arrow/go/v14/arrow/memory"
|
||||
"github.com/apache/arrow/go/v14/parquet"
|
||||
"github.com/apache/arrow/go/v14/parquet/compress"
|
||||
"github.com/apache/arrow/go/v14/parquet/pqarrow"
|
||||
)
|
||||
|
||||
func BackupOrders(sqlDB *sql.DB) (string, error) {
|
||||
|
||||
rows, err := sqlDB.Query(`
|
||||
SELECT DISTINCT a.orderheaderid, a.applocationid,
|
||||
a.tenantid, a.locationid, a.partnerid, a.configid, a.categoryid, a.subcategoryid, a.moduleid,
|
||||
a.orderid, a.orderstatus, a.orderdate, a.ordernotes, a.itemcount, a.deliverytime AS deliverydate,
|
||||
a.pending, a.processing, a.ready, a.delivered AS completed, a.cancelled,
|
||||
a.deliverycharge, a.kms,
|
||||
a.customerid, a.pickupaddress, a.pickuplat, a.pickuplong,
|
||||
a.pickupcustomer, a.pickupcontactno, a.pickuplocation as pickupsuburb, a.pickupcity,
|
||||
a.deliveryid AS deliverycustomerid, a.deliveryaddress, a.deliverylat, a.deliverylong, a.deliverytype,
|
||||
a.deliverycustomer, a.deliverycontactno, a.deliverylocation as deliverysuburb, a.deliverycity, a.paymenttype, a.smsdelivery, a.orderamount, a.quantity, a.collectionamt,
|
||||
b.tenantname, b.tenanttoken, b.primarycontact AS tenantcontactno, b.postcode AS tenantpostcode, b.suburb AS tenantsuburb, b.city AS tenantcity, b.address AS tenantaddress,
|
||||
c.locationname, c.contactno AS locationcontactno, c.postcode AS locationpostcode, c.suburb AS locationsuburb, c.city AS locationcity, c.address AS locationaddress,
|
||||
d.locationname AS applocation, f.slab, f.pricingdate, f.baseprice, f.minkm, f.priceperkm, f.maxkm, f.orders, f.othercharges, f.surgecharges
|
||||
FROM orders a
|
||||
INNER JOIN tenants b ON a.tenantid = b.tenantid
|
||||
INNER JOIN tenantlocations c ON a.locationid = c.locationid
|
||||
INNER JOIN app_location d ON a.applocationid = d.applocationid
|
||||
INNER JOIN app_locationconfig e ON d.applocationid = e.applocationid
|
||||
LEFT JOIN tenantpricing f ON a.tenantid = f.tenantid
|
||||
WHERE a.orderdate >= NOW() - INTERVAL '3 months'
|
||||
`)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// ---------- Arrow Schema (aligned with SQL SELECT — 67 columns) ----------
|
||||
schema := arrow.NewSchema([]arrow.Field{
|
||||
// core ids
|
||||
{Name: "orderheaderid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "applocationid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "tenantid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "locationid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "partnerid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "configid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "categoryid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "subcategoryid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "moduleid", Type: arrow.PrimitiveTypes.Int32},
|
||||
// order meta
|
||||
{Name: "orderid", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "orderstatus", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "orderdate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
{Name: "ordernotes", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "itemcount", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "deliverydate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
// lifecycle
|
||||
{Name: "pending", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "processing", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "ready", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "completed", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "cancelled", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// charges & distance
|
||||
{Name: "deliverycharge", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "kms", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// pickup
|
||||
{Name: "customerid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "pickupaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplat", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplong", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickupcustomer", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickupcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickupcity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// delivery
|
||||
{Name: "deliverycustomerid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "deliveryaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylat", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylong", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverytype", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverycustomer", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverycontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverycity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// payment
|
||||
{Name: "paymenttype", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "smsdelivery", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "orderamount", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "quantity", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "collectionamt", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
// tenant
|
||||
{Name: "tenantname", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenanttoken", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantpostcode", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantsuburb", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantcity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// location
|
||||
{Name: "locationname", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationpostcode", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationsuburb", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationcity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// app location + pricing
|
||||
{Name: "applocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "slab", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pricingdate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
{Name: "baseprice", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "minkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "priceperkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "maxkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "pricingorders", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "othercharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "surgecharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
}, nil)
|
||||
|
||||
pool := memory.NewGoAllocator()
|
||||
builder := array.NewRecordBuilder(pool, schema)
|
||||
defer builder.Release()
|
||||
|
||||
// ---------- Scan & Append ----------
|
||||
for rows.Next() {
|
||||
|
||||
var (
|
||||
orderheaderid, applocationid, tenantid, locationid, partnerid int32
|
||||
configid, categoryid, subcategoryid, moduleid int32
|
||||
|
||||
orderid, orderstatus, ordernotes sql.NullString
|
||||
pending, processing, ready, completed, cancelled sql.NullString
|
||||
pickupaddress, pickuplat, pickuplong sql.NullString
|
||||
pickupcustomer, pickupcontactno, pickupsuburb, pickupcity sql.NullString
|
||||
deliveryaddress, deliverylat, deliverylong sql.NullString
|
||||
deliverytype, deliverycustomer, deliverycontactno sql.NullString
|
||||
deliverysuburb, deliverycity sql.NullString
|
||||
kms sql.NullString
|
||||
|
||||
itemcount, paymenttype, smsdelivery, quantity sql.NullInt32
|
||||
customerid, deliverycustomerid int32
|
||||
orderamount, deliverycharge, collectionamt sql.NullFloat64
|
||||
|
||||
orderdate, deliverydate sql.NullTime
|
||||
|
||||
tenantname, tenanttoken, tenantcontactno sql.NullString
|
||||
tenantpostcode, tenantsuburb, tenantcity, tenantaddress sql.NullString
|
||||
|
||||
locationname, locationcontactno, locationpostcode sql.NullString
|
||||
locationsuburb, locationcity, locationaddress sql.NullString
|
||||
|
||||
applocation sql.NullString
|
||||
slab sql.NullString
|
||||
pricingdate sql.NullTime
|
||||
baseprice, minkm, priceperkm, maxkm sql.NullFloat64
|
||||
pricingorders sql.NullInt32
|
||||
othercharges, surgecharges sql.NullFloat64
|
||||
)
|
||||
|
||||
if err := rows.Scan(
|
||||
// core ids
|
||||
&orderheaderid, &applocationid,
|
||||
&tenantid, &locationid, &partnerid, &configid, &categoryid, &subcategoryid, &moduleid,
|
||||
// order meta
|
||||
&orderid, &orderstatus, &orderdate, &ordernotes, &itemcount, &deliverydate,
|
||||
// lifecycle
|
||||
&pending, &processing, &ready, &completed, &cancelled,
|
||||
// charges & distance
|
||||
&deliverycharge, &kms,
|
||||
// pickup
|
||||
&customerid, &pickupaddress, &pickuplat, &pickuplong,
|
||||
&pickupcustomer, &pickupcontactno, &pickupsuburb, &pickupcity,
|
||||
// delivery
|
||||
&deliverycustomerid, &deliveryaddress, &deliverylat, &deliverylong,
|
||||
&deliverytype, &deliverycustomer, &deliverycontactno,
|
||||
&deliverysuburb, &deliverycity,
|
||||
// payment
|
||||
&paymenttype, &smsdelivery, &orderamount, &quantity, &collectionamt,
|
||||
// tenant
|
||||
&tenantname, &tenanttoken, &tenantcontactno,
|
||||
&tenantpostcode, &tenantsuburb, &tenantcity, &tenantaddress,
|
||||
// location
|
||||
&locationname, &locationcontactno, &locationpostcode,
|
||||
&locationsuburb, &locationcity, &locationaddress,
|
||||
// app location + pricing
|
||||
&applocation,
|
||||
&slab, &pricingdate, &baseprice, &minkm, &priceperkm,
|
||||
&maxkm, &pricingorders, &othercharges, &surgecharges,
|
||||
); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// core ids (not null)
|
||||
builder.Field(0).(*array.Int32Builder).Append(orderheaderid)
|
||||
builder.Field(1).(*array.Int32Builder).Append(applocationid)
|
||||
builder.Field(2).(*array.Int32Builder).Append(tenantid)
|
||||
builder.Field(3).(*array.Int32Builder).Append(locationid)
|
||||
builder.Field(4).(*array.Int32Builder).Append(partnerid)
|
||||
builder.Field(5).(*array.Int32Builder).Append(configid)
|
||||
builder.Field(6).(*array.Int32Builder).Append(categoryid)
|
||||
builder.Field(7).(*array.Int32Builder).Append(subcategoryid)
|
||||
builder.Field(8).(*array.Int32Builder).Append(moduleid)
|
||||
|
||||
// order meta
|
||||
appendString(builder.Field(9), orderid)
|
||||
appendString(builder.Field(10), orderstatus)
|
||||
if orderdate.Valid {
|
||||
builder.Field(11).(*array.TimestampBuilder).Append(arrow.Timestamp(orderdate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(11).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
appendString(builder.Field(12), ordernotes)
|
||||
if itemcount.Valid {
|
||||
builder.Field(13).(*array.Int32Builder).Append(itemcount.Int32)
|
||||
} else {
|
||||
builder.Field(13).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if deliverydate.Valid {
|
||||
builder.Field(14).(*array.TimestampBuilder).Append(arrow.Timestamp(deliverydate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(14).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
|
||||
// lifecycle
|
||||
appendString(builder.Field(15), pending)
|
||||
appendString(builder.Field(16), processing)
|
||||
appendString(builder.Field(17), ready)
|
||||
appendString(builder.Field(18), completed)
|
||||
appendString(builder.Field(19), cancelled)
|
||||
|
||||
// charges & distance
|
||||
if deliverycharge.Valid {
|
||||
builder.Field(20).(*array.Float64Builder).Append(deliverycharge.Float64)
|
||||
} else {
|
||||
builder.Field(20).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
appendString(builder.Field(21), kms)
|
||||
|
||||
// pickup
|
||||
builder.Field(22).(*array.Int32Builder).Append(customerid)
|
||||
appendString(builder.Field(23), pickupaddress)
|
||||
appendString(builder.Field(24), pickuplat)
|
||||
appendString(builder.Field(25), pickuplong)
|
||||
appendString(builder.Field(26), pickupcustomer)
|
||||
appendString(builder.Field(27), pickupcontactno)
|
||||
appendString(builder.Field(28), pickupsuburb)
|
||||
appendString(builder.Field(29), pickupcity)
|
||||
|
||||
// delivery
|
||||
builder.Field(30).(*array.Int32Builder).Append(deliverycustomerid)
|
||||
appendString(builder.Field(31), deliveryaddress)
|
||||
appendString(builder.Field(32), deliverylat)
|
||||
appendString(builder.Field(33), deliverylong)
|
||||
appendString(builder.Field(34), deliverytype)
|
||||
appendString(builder.Field(35), deliverycustomer)
|
||||
appendString(builder.Field(36), deliverycontactno)
|
||||
appendString(builder.Field(37), deliverysuburb)
|
||||
appendString(builder.Field(38), deliverycity)
|
||||
|
||||
// payment
|
||||
if paymenttype.Valid {
|
||||
builder.Field(39).(*array.Int32Builder).Append(paymenttype.Int32)
|
||||
} else {
|
||||
builder.Field(39).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if smsdelivery.Valid {
|
||||
builder.Field(40).(*array.Int32Builder).Append(smsdelivery.Int32)
|
||||
} else {
|
||||
builder.Field(40).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if orderamount.Valid {
|
||||
builder.Field(41).(*array.Float64Builder).Append(orderamount.Float64)
|
||||
} else {
|
||||
builder.Field(41).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if quantity.Valid {
|
||||
builder.Field(42).(*array.Int32Builder).Append(quantity.Int32)
|
||||
} else {
|
||||
builder.Field(42).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if collectionamt.Valid {
|
||||
builder.Field(43).(*array.Float64Builder).Append(collectionamt.Float64)
|
||||
} else {
|
||||
builder.Field(43).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
// tenant
|
||||
appendString(builder.Field(44), tenantname)
|
||||
appendString(builder.Field(45), tenanttoken)
|
||||
appendString(builder.Field(46), tenantcontactno)
|
||||
appendString(builder.Field(47), tenantpostcode)
|
||||
appendString(builder.Field(48), tenantsuburb)
|
||||
appendString(builder.Field(49), tenantcity)
|
||||
appendString(builder.Field(50), tenantaddress)
|
||||
|
||||
// location
|
||||
appendString(builder.Field(51), locationname)
|
||||
appendString(builder.Field(52), locationcontactno)
|
||||
appendString(builder.Field(53), locationpostcode)
|
||||
appendString(builder.Field(54), locationsuburb)
|
||||
appendString(builder.Field(55), locationcity)
|
||||
appendString(builder.Field(56), locationaddress)
|
||||
|
||||
// app location + pricing
|
||||
appendString(builder.Field(57), applocation)
|
||||
appendString(builder.Field(58), slab)
|
||||
if pricingdate.Valid {
|
||||
builder.Field(59).(*array.TimestampBuilder).Append(arrow.Timestamp(pricingdate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(59).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
if baseprice.Valid {
|
||||
builder.Field(60).(*array.Float64Builder).Append(baseprice.Float64)
|
||||
} else {
|
||||
builder.Field(60).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if minkm.Valid {
|
||||
builder.Field(61).(*array.Float64Builder).Append(minkm.Float64)
|
||||
} else {
|
||||
builder.Field(61).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if priceperkm.Valid {
|
||||
builder.Field(62).(*array.Float64Builder).Append(priceperkm.Float64)
|
||||
} else {
|
||||
builder.Field(62).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if maxkm.Valid {
|
||||
builder.Field(63).(*array.Float64Builder).Append(maxkm.Float64)
|
||||
} else {
|
||||
builder.Field(63).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if pricingorders.Valid {
|
||||
builder.Field(64).(*array.Int32Builder).Append(pricingorders.Int32)
|
||||
} else {
|
||||
builder.Field(64).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if othercharges.Valid {
|
||||
builder.Field(65).(*array.Float64Builder).Append(othercharges.Float64)
|
||||
} else {
|
||||
builder.Field(65).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if surgecharges.Valid {
|
||||
builder.Field(66).(*array.Float64Builder).Append(surgecharges.Float64)
|
||||
} else {
|
||||
builder.Field(66).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
}
|
||||
|
||||
record := builder.NewRecord()
|
||||
defer record.Release()
|
||||
|
||||
_ = os.MkdirAll("backups/parquet/orders", 0755)
|
||||
|
||||
filePath := "backups/parquet/orders/orders_last_3_months_" +
|
||||
time.Now().Format("2006_01_02") + ".parquet"
|
||||
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
parquetProps := parquet.NewWriterProperties(
|
||||
parquet.WithCompression(compress.Codecs.Snappy),
|
||||
)
|
||||
|
||||
writer, err := pqarrow.NewFileWriter(
|
||||
schema,
|
||||
file,
|
||||
parquetProps,
|
||||
pqarrow.ArrowWriterProperties{},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
if err := writer.Write(record); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filePath, nil
|
||||
}
|
||||
Reference in New Issue
Block a user