intial commit
This commit is contained in:
156
internal/backup/mysql_to_parquet.go
Normal file
156
internal/backup/mysql_to_parquet.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/apache/arrow/go/v14/arrow"
|
||||
"github.com/apache/arrow/go/v14/arrow/array"
|
||||
"github.com/apache/arrow/go/v14/arrow/memory"
|
||||
"github.com/apache/arrow/go/v14/parquet"
|
||||
"github.com/apache/arrow/go/v14/parquet/compress"
|
||||
"github.com/apache/arrow/go/v14/parquet/pqarrow"
|
||||
)
|
||||
|
||||
func BackupDeliveryLogs(sqlDB *sql.DB) (string, error) {
|
||||
|
||||
rows, err := sqlDB.Query(`
|
||||
SELECT
|
||||
logid, logdate, deliveryid, orderheaderid,
|
||||
tenantid, locationid, userid, partnerid,
|
||||
orderid, orderstatus, latitude, longitude,
|
||||
logstatus, created, updated, firstname, lastname
|
||||
FROM deliverylogs
|
||||
WHERE logdate >= NOW() - INTERVAL '3 months'
|
||||
`)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Arrow schema (exact match with MySQL table)
|
||||
schema := arrow.NewSchema([]arrow.Field{
|
||||
{Name: "logid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "logdate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
{Name: "deliveryid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "orderheaderid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "tenantid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "locationid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "userid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "partnerid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "orderid", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "orderstatus", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "latitude", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "longitude", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "logstatus", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "created", Type: arrow.FixedWidthTypes.Timestamp_ms},
|
||||
{Name: "updated", Type: arrow.FixedWidthTypes.Timestamp_ms},
|
||||
{Name: "firstname", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "lastname", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
}, nil)
|
||||
|
||||
pool := memory.NewGoAllocator()
|
||||
builder := array.NewRecordBuilder(pool, schema)
|
||||
defer builder.Release()
|
||||
|
||||
for rows.Next() {
|
||||
|
||||
var (
|
||||
logid, deliveryid, orderheaderid int32
|
||||
tenantid, locationid, userid int32
|
||||
partnerid, logstatus int32
|
||||
|
||||
logdate, created, updated sql.NullTime
|
||||
orderid, orderstatus sql.NullString
|
||||
latitude, longitude sql.NullString
|
||||
firstname, lastname sql.NullString
|
||||
)
|
||||
|
||||
if err := rows.Scan(
|
||||
&logid, &logdate, &deliveryid, &orderheaderid,
|
||||
&tenantid, &locationid, &userid, &partnerid,
|
||||
&orderid, &orderstatus, &latitude, &longitude,
|
||||
&logstatus, &created, &updated, &firstname, &lastname,
|
||||
); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
builder.Field(0).(*array.Int32Builder).Append(logid)
|
||||
|
||||
if logdate.Valid {
|
||||
builder.Field(1).(*array.TimestampBuilder).
|
||||
Append(arrow.Timestamp(logdate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(1).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
|
||||
builder.Field(2).(*array.Int32Builder).Append(deliveryid)
|
||||
builder.Field(3).(*array.Int32Builder).Append(orderheaderid)
|
||||
builder.Field(4).(*array.Int32Builder).Append(tenantid)
|
||||
builder.Field(5).(*array.Int32Builder).Append(locationid)
|
||||
builder.Field(6).(*array.Int32Builder).Append(userid)
|
||||
builder.Field(7).(*array.Int32Builder).Append(partnerid)
|
||||
|
||||
appendString(builder.Field(8), orderid)
|
||||
appendString(builder.Field(9), orderstatus)
|
||||
appendString(builder.Field(10), latitude)
|
||||
appendString(builder.Field(11), longitude)
|
||||
|
||||
builder.Field(12).(*array.Int32Builder).Append(logstatus)
|
||||
|
||||
builder.Field(13).(*array.TimestampBuilder).
|
||||
Append(arrow.Timestamp(created.Time.UnixMilli()))
|
||||
builder.Field(14).(*array.TimestampBuilder).
|
||||
Append(arrow.Timestamp(updated.Time.UnixMilli()))
|
||||
|
||||
appendString(builder.Field(15), firstname)
|
||||
appendString(builder.Field(16), lastname)
|
||||
}
|
||||
|
||||
record := builder.NewRecord()
|
||||
defer record.Release()
|
||||
|
||||
_ = os.MkdirAll("backups/parquet/deliverylogs", 0755)
|
||||
|
||||
filePath := "backups/parquet/deliverylogs/deliverylogs_last_3_months_" +
|
||||
time.Now().Format("2006_01_02") + ".parquet"
|
||||
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// ✅ Correct Parquet + Arrow writer props (Arrow v14)
|
||||
parquetProps := parquet.NewWriterProperties(
|
||||
parquet.WithCompression(compress.Codecs.Snappy),
|
||||
)
|
||||
|
||||
writer, err := pqarrow.NewFileWriter(
|
||||
schema,
|
||||
file,
|
||||
parquetProps,
|
||||
pqarrow.ArrowWriterProperties{}, // zero-value is valid
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
if err := writer.Write(record); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
// helper for nullable strings
|
||||
func appendString(b array.Builder, v sql.NullString) {
|
||||
sb := b.(*array.StringBuilder)
|
||||
if v.Valid {
|
||||
sb.Append(v.String)
|
||||
} else {
|
||||
sb.AppendNull()
|
||||
}
|
||||
}
|
||||
764
internal/backup/mysql_to_parquet_deliveries.go
Normal file
764
internal/backup/mysql_to_parquet_deliveries.go
Normal file
@@ -0,0 +1,764 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/apache/arrow/go/v14/arrow"
|
||||
"github.com/apache/arrow/go/v14/arrow/array"
|
||||
"github.com/apache/arrow/go/v14/arrow/memory"
|
||||
"github.com/apache/arrow/go/v14/parquet"
|
||||
"github.com/apache/arrow/go/v14/parquet/compress"
|
||||
"github.com/apache/arrow/go/v14/parquet/pqarrow"
|
||||
)
|
||||
|
||||
func BackupDeliveries(sqlDB *sql.DB) (string, error) {
|
||||
|
||||
rows, err := sqlDB.Query(`
|
||||
SELECT DISTINCT a.deliveryid,a.applocationid,f.locationname AS applocation,a.orderheaderid,a.configid,a.tenantid,a.partnerid,a.locationid,a.userid,a.categoryid,a.subcategoryid,a.moduleid,
|
||||
a.orderid,a.deliverydate,a.orderstatus,a.assigntime,a.starttime,a.arrivaltime,a.pickuptime,a.deliverytime,a.canceltime,a.acceptedtime,
|
||||
a.customerid,a.pickupcustomer,a.pickupcontactno,a.pickuplocationid,a.pickupaddress,a.pickuplocation,a.pickuplat,a.pickuplon,
|
||||
a.deliverycustomerid,a.deliverycustomer,a.deliverycontactno,a.deliverylocationid,a.deliveryaddress,a.deliverylocation,
|
||||
a.droplat,a.droplon,a.deliverylat,a.deliverylong,
|
||||
a.riderslat,a.riderslon,a.deliveryamt,a.kms,a.actualkms,a.riderkms,a.deliverycharges,a.deliverytype,a.paymenttype,a.smsdelivery,a.quantity,a.collectionamt,a.collectedamt,a.collectionstatus,
|
||||
a.notes,a.ordernotes,a.step,a.eta,a.previouskms,a.cumulativekms,a.ridertime,b.tenantname,b.primarycontact as tenantcontactno,b.tenanttoken,b.suburb as tenantsuburb,b.city as tenantcity,
|
||||
b.address AS tenantaddress, CONCAT(c.firstname, ' ', c.lastname) AS ridername,c.userfcmtoken,c.contactno as ridercontact,e.locationname,e.suburb AS locationsuburb,e.contactno AS locationcontactno,e.address AS locationaddress,
|
||||
h.slab, h.pricingdate, h.baseprice, h.minkm, h.priceperkm, h.maxkm, h.orders, h.othercharges, h.surgecharges
|
||||
FROM deliveries a
|
||||
INNER JOIN tenants b ON a.tenantid=b.tenantid
|
||||
INNER JOIN app_users c ON a.userid=c.userid
|
||||
INNER JOIN tenantlocations e ON a.locationid=e.locationid
|
||||
INNER JOIN app_location f ON a.applocationid = f.applocationid
|
||||
INNER JOIN app_locationconfig g ON f.applocationid = g.applocationid
|
||||
LEFT JOIN tenantpricing h ON a.tenantid = h.tenantid
|
||||
WHERE a.deliverydate >= NOW() - INTERVAL '3 months'
|
||||
`)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// ✅ REAL schema (no placeholders)
|
||||
schema := arrow.NewSchema([]arrow.Field{
|
||||
{Name: "deliveryid", Type: arrow.PrimitiveTypes.Int32},
|
||||
|
||||
{Name: "applocationid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "applocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "orderheaderid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "configid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "tenantid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "partnerid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "locationid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "userid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "categoryid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "subcategoryid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "moduleid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
|
||||
{Name: "orderid", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverydate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
{Name: "orderstatus", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "assigntime", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "starttime", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "arrivaltime", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuptime", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverytime", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "canceltime", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "acceptedtime", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
|
||||
{Name: "customerid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "pickupcustomer", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickupcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplocationid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "pickupaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplat", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplon", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
|
||||
{Name: "deliverycustomerid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "deliverycustomer", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverycontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylocationid", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "deliveryaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "droplat", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "droplon", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylat", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylong", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
|
||||
{Name: "riderslat", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "riderslon", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliveryamt", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "kms", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "actualkms", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "riderkms", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverycharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "deliverytype", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "paymenttype", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "smsdelivery", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "quantity", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "collectionamt", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "collectedamt", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "collectionstatus", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
|
||||
{Name: "notes", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "ordernotes", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "step", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "eta", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "previouskms", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "cumulativekms", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "ridertime", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
|
||||
{Name: "tenantname", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenanttoken", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantsuburb", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantcity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
|
||||
{Name: "ridername", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "userfcmtoken", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "ridercontact", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
|
||||
{Name: "locationname", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationsuburb", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
|
||||
{Name: "slab", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pricingdate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
{Name: "baseprice", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "minkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "priceperkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "maxkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "orders", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "othercharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "surgecharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
}, nil)
|
||||
|
||||
pool := memory.NewGoAllocator()
|
||||
builder := array.NewRecordBuilder(pool, schema)
|
||||
defer builder.Release()
|
||||
|
||||
for rows.Next() {
|
||||
|
||||
var (
|
||||
deliveryid int32
|
||||
orderheaderid, configid, applocationid sql.NullInt32
|
||||
tenantid, locationid, partnerid, moduleid, userid sql.NullInt32
|
||||
categoryid, subcategoryid sql.NullInt32
|
||||
orderid, orderstatus, assigntime, starttime sql.NullString
|
||||
arrivaltime, pickuptime, acceptedtime, deliverytime sql.NullString
|
||||
canceltime sql.NullString
|
||||
|
||||
customerid, pickuplocationid sql.NullInt32
|
||||
pickupcustomer, pickupaddress, pickuplocation sql.NullString
|
||||
pickupcontactno, pickuplat, pickuplon sql.NullString
|
||||
deliverycustomerid, deliverylocationid sql.NullInt32
|
||||
deliverycustomer, deliverycontactno, deliveryaddress sql.NullString
|
||||
deliverylocation, droplat, droplon sql.NullString
|
||||
deliverylat, deliverylong, riderslat, riderslon sql.NullString
|
||||
|
||||
kms, riderkms, actualkms sql.NullString
|
||||
deliverycharges, deliveryamt sql.NullFloat64
|
||||
notes, deliverytype, ordernotes sql.NullString
|
||||
paymenttype sql.NullInt32
|
||||
|
||||
smsdelivery sql.NullInt32
|
||||
previouskms, cumulativekms, step, quantity sql.NullInt32
|
||||
collectionamt, collectedamt sql.NullFloat64
|
||||
collectionstatus sql.NullInt32
|
||||
eta sql.NullString
|
||||
ridertime sql.NullInt32
|
||||
deliverydate sql.NullTime
|
||||
|
||||
applocation sql.NullString
|
||||
|
||||
tenantname, tenantcontactno, tenanttoken sql.NullString
|
||||
tenantsuburb, tenantcity, tenantaddress sql.NullString
|
||||
|
||||
ridername, userfcmtoken, ridercontact sql.NullString
|
||||
|
||||
locationname, locationsuburb sql.NullString
|
||||
locationcontactno, locationaddress sql.NullString
|
||||
|
||||
slab sql.NullString
|
||||
pricingdate sql.NullTime
|
||||
baseprice, minkm, priceperkm, maxkm sql.NullFloat64
|
||||
pricingorders sql.NullInt32
|
||||
othercharges, surgecharges sql.NullFloat64
|
||||
)
|
||||
|
||||
if err := rows.Scan(
|
||||
&deliveryid, &applocationid, &applocation,
|
||||
&orderheaderid, &configid, &tenantid, &partnerid,
|
||||
&locationid, &userid, &categoryid, &subcategoryid, &moduleid,
|
||||
|
||||
&orderid, &deliverydate, &orderstatus,
|
||||
&assigntime, &starttime, &arrivaltime, &pickuptime,
|
||||
&deliverytime, &canceltime, &acceptedtime,
|
||||
|
||||
&customerid, &pickupcustomer, &pickupcontactno,
|
||||
&pickuplocationid, &pickupaddress, &pickuplocation,
|
||||
&pickuplat, &pickuplon,
|
||||
|
||||
&deliverycustomerid, &deliverycustomer, &deliverycontactno,
|
||||
&deliverylocationid, &deliveryaddress, &deliverylocation,
|
||||
&droplat, &droplon, &deliverylat, &deliverylong,
|
||||
|
||||
&riderslat, &riderslon,
|
||||
&deliveryamt, &kms, &actualkms, &riderkms,
|
||||
&deliverycharges, &deliverytype, &paymenttype,
|
||||
&smsdelivery, &quantity, &collectionamt,
|
||||
&collectedamt, &collectionstatus,
|
||||
|
||||
¬es, &ordernotes, &step, &eta,
|
||||
&previouskms, &cumulativekms, &ridertime,
|
||||
|
||||
&tenantname, &tenantcontactno, &tenanttoken,
|
||||
&tenantsuburb, &tenantcity, &tenantaddress,
|
||||
|
||||
&ridername, &userfcmtoken, &ridercontact,
|
||||
|
||||
&locationname, &locationsuburb, &locationcontactno, &locationaddress,
|
||||
|
||||
&slab, &pricingdate, &baseprice, &minkm,
|
||||
&priceperkm, &maxkm, &pricingorders,
|
||||
&othercharges, &surgecharges,
|
||||
); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// ✅ Append safely
|
||||
builder.Field(0).(*array.Int32Builder).Append(deliveryid)
|
||||
|
||||
if applocationid.Valid {
|
||||
builder.Field(1).(*array.Int32Builder).Append(applocationid.Int32)
|
||||
} else {
|
||||
builder.Field(1).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if applocation.Valid {
|
||||
builder.Field(2).(*array.StringBuilder).Append(applocation.String)
|
||||
} else {
|
||||
builder.Field(2).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if orderheaderid.Valid {
|
||||
builder.Field(3).(*array.Int32Builder).Append(orderheaderid.Int32)
|
||||
} else {
|
||||
builder.Field(3).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if configid.Valid {
|
||||
builder.Field(4).(*array.Int32Builder).Append(configid.Int32)
|
||||
} else {
|
||||
builder.Field(4).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if tenantid.Valid {
|
||||
builder.Field(5).(*array.Int32Builder).Append(tenantid.Int32)
|
||||
} else {
|
||||
builder.Field(5).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if partnerid.Valid {
|
||||
builder.Field(6).(*array.Int32Builder).Append(partnerid.Int32)
|
||||
} else {
|
||||
builder.Field(6).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if locationid.Valid {
|
||||
builder.Field(7).(*array.Int32Builder).Append(locationid.Int32)
|
||||
} else {
|
||||
builder.Field(7).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if userid.Valid {
|
||||
builder.Field(8).(*array.Int32Builder).Append(userid.Int32)
|
||||
} else {
|
||||
builder.Field(8).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if categoryid.Valid {
|
||||
builder.Field(9).(*array.Int32Builder).Append(categoryid.Int32)
|
||||
} else {
|
||||
builder.Field(9).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if subcategoryid.Valid {
|
||||
builder.Field(10).(*array.Int32Builder).Append(subcategoryid.Int32)
|
||||
} else {
|
||||
builder.Field(10).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if moduleid.Valid {
|
||||
builder.Field(11).(*array.Int32Builder).Append(moduleid.Int32)
|
||||
} else {
|
||||
builder.Field(11).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if orderid.Valid {
|
||||
builder.Field(12).(*array.StringBuilder).Append(orderid.String)
|
||||
} else {
|
||||
builder.Field(12).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverydate.Valid {
|
||||
builder.Field(13).(*array.TimestampBuilder).
|
||||
Append(arrow.Timestamp(deliverydate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(13).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if orderstatus.Valid {
|
||||
builder.Field(14).(*array.StringBuilder).Append(orderstatus.String)
|
||||
} else {
|
||||
builder.Field(14).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if assigntime.Valid {
|
||||
builder.Field(15).(*array.StringBuilder).Append(assigntime.String)
|
||||
} else {
|
||||
builder.Field(15).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if starttime.Valid {
|
||||
builder.Field(16).(*array.StringBuilder).Append(starttime.String)
|
||||
} else {
|
||||
builder.Field(16).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if arrivaltime.Valid {
|
||||
builder.Field(17).(*array.StringBuilder).Append(arrivaltime.String)
|
||||
} else {
|
||||
builder.Field(17).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if pickuptime.Valid {
|
||||
builder.Field(18).(*array.StringBuilder).Append(pickuptime.String)
|
||||
} else {
|
||||
builder.Field(18).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverytime.Valid {
|
||||
builder.Field(19).(*array.StringBuilder).Append(deliverytime.String)
|
||||
} else {
|
||||
builder.Field(19).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if canceltime.Valid {
|
||||
builder.Field(20).(*array.StringBuilder).Append(canceltime.String)
|
||||
} else {
|
||||
builder.Field(20).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if acceptedtime.Valid {
|
||||
builder.Field(21).(*array.StringBuilder).Append(acceptedtime.String)
|
||||
} else {
|
||||
builder.Field(21).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if customerid.Valid {
|
||||
builder.Field(22).(*array.Int32Builder).Append(customerid.Int32)
|
||||
} else {
|
||||
builder.Field(22).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if pickupcustomer.Valid {
|
||||
builder.Field(23).(*array.StringBuilder).Append(pickupcustomer.String)
|
||||
} else {
|
||||
builder.Field(23).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if pickupcontactno.Valid {
|
||||
builder.Field(24).(*array.StringBuilder).Append(pickupcontactno.String)
|
||||
} else {
|
||||
builder.Field(24).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if pickuplocationid.Valid {
|
||||
builder.Field(25).(*array.Int32Builder).Append(pickuplocationid.Int32)
|
||||
} else {
|
||||
builder.Field(25).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if pickupaddress.Valid {
|
||||
builder.Field(26).(*array.StringBuilder).Append(pickupaddress.String)
|
||||
} else {
|
||||
builder.Field(26).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if pickuplocation.Valid {
|
||||
builder.Field(27).(*array.StringBuilder).Append(pickuplocation.String)
|
||||
} else {
|
||||
builder.Field(27).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if pickuplat.Valid {
|
||||
builder.Field(28).(*array.StringBuilder).Append(pickuplat.String)
|
||||
} else {
|
||||
builder.Field(28).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if pickuplon.Valid {
|
||||
builder.Field(29).(*array.StringBuilder).Append(pickuplon.String)
|
||||
} else {
|
||||
builder.Field(29).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverycustomerid.Valid {
|
||||
builder.Field(30).(*array.Int32Builder).Append(deliverycustomerid.Int32)
|
||||
} else {
|
||||
builder.Field(30).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverycustomer.Valid {
|
||||
builder.Field(31).(*array.StringBuilder).Append(deliverycustomer.String)
|
||||
} else {
|
||||
builder.Field(31).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverycontactno.Valid {
|
||||
builder.Field(32).(*array.StringBuilder).Append(deliverycontactno.String)
|
||||
} else {
|
||||
builder.Field(32).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverylocationid.Valid {
|
||||
builder.Field(33).(*array.Int32Builder).Append(deliverylocationid.Int32)
|
||||
} else {
|
||||
builder.Field(33).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if deliveryaddress.Valid {
|
||||
builder.Field(34).(*array.StringBuilder).Append(deliveryaddress.String)
|
||||
} else {
|
||||
builder.Field(34).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverylocation.Valid {
|
||||
builder.Field(35).(*array.StringBuilder).Append(deliverylocation.String)
|
||||
} else {
|
||||
builder.Field(35).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if droplat.Valid {
|
||||
builder.Field(36).(*array.StringBuilder).Append(droplat.String)
|
||||
} else {
|
||||
builder.Field(36).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if droplon.Valid {
|
||||
builder.Field(37).(*array.StringBuilder).Append(droplon.String)
|
||||
} else {
|
||||
builder.Field(37).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverylat.Valid {
|
||||
builder.Field(38).(*array.StringBuilder).Append(deliverylat.String)
|
||||
} else {
|
||||
builder.Field(38).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverylong.Valid {
|
||||
builder.Field(39).(*array.StringBuilder).Append(deliverylong.String)
|
||||
} else {
|
||||
builder.Field(39).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if riderslat.Valid {
|
||||
builder.Field(40).(*array.StringBuilder).Append(riderslat.String)
|
||||
} else {
|
||||
builder.Field(40).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if riderslon.Valid {
|
||||
builder.Field(41).(*array.StringBuilder).Append(riderslon.String)
|
||||
} else {
|
||||
builder.Field(41).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliveryamt.Valid {
|
||||
builder.Field(42).(*array.Float64Builder).Append(deliveryamt.Float64)
|
||||
} else {
|
||||
builder.Field(42).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if kms.Valid {
|
||||
builder.Field(43).(*array.StringBuilder).Append(kms.String)
|
||||
} else {
|
||||
builder.Field(43).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if actualkms.Valid {
|
||||
builder.Field(44).(*array.StringBuilder).Append(actualkms.String)
|
||||
} else {
|
||||
builder.Field(44).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if riderkms.Valid {
|
||||
builder.Field(45).(*array.StringBuilder).Append(riderkms.String)
|
||||
} else {
|
||||
builder.Field(45).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverycharges.Valid {
|
||||
builder.Field(46).(*array.Float64Builder).Append(deliverycharges.Float64)
|
||||
} else {
|
||||
builder.Field(46).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if deliverytype.Valid {
|
||||
builder.Field(47).(*array.StringBuilder).Append(deliverytype.String)
|
||||
} else {
|
||||
builder.Field(47).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if paymenttype.Valid {
|
||||
builder.Field(48).(*array.Int32Builder).Append(paymenttype.Int32)
|
||||
} else {
|
||||
builder.Field(48).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if smsdelivery.Valid {
|
||||
builder.Field(49).(*array.Int32Builder).Append(smsdelivery.Int32)
|
||||
} else {
|
||||
builder.Field(49).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if quantity.Valid {
|
||||
builder.Field(50).(*array.Int32Builder).Append(quantity.Int32)
|
||||
} else {
|
||||
builder.Field(50).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if collectionamt.Valid {
|
||||
builder.Field(51).(*array.Float64Builder).Append(collectionamt.Float64)
|
||||
} else {
|
||||
builder.Field(51).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if collectedamt.Valid {
|
||||
builder.Field(52).(*array.Float64Builder).Append(collectedamt.Float64)
|
||||
} else {
|
||||
builder.Field(52).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if collectionstatus.Valid {
|
||||
builder.Field(53).(*array.Int32Builder).Append(collectionstatus.Int32)
|
||||
} else {
|
||||
builder.Field(53).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if notes.Valid {
|
||||
builder.Field(54).(*array.StringBuilder).Append(notes.String)
|
||||
} else {
|
||||
builder.Field(54).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if ordernotes.Valid {
|
||||
builder.Field(55).(*array.StringBuilder).Append(ordernotes.String)
|
||||
} else {
|
||||
builder.Field(55).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if step.Valid {
|
||||
builder.Field(56).(*array.Int32Builder).Append(step.Int32)
|
||||
} else {
|
||||
builder.Field(56).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if eta.Valid {
|
||||
builder.Field(57).(*array.StringBuilder).Append(eta.String)
|
||||
} else {
|
||||
builder.Field(57).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if previouskms.Valid {
|
||||
builder.Field(58).(*array.Int32Builder).Append(previouskms.Int32)
|
||||
} else {
|
||||
builder.Field(58).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if cumulativekms.Valid {
|
||||
builder.Field(59).(*array.Int32Builder).Append(cumulativekms.Int32)
|
||||
} else {
|
||||
builder.Field(59).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if ridertime.Valid {
|
||||
builder.Field(60).(*array.Int32Builder).Append(ridertime.Int32)
|
||||
} else {
|
||||
builder.Field(60).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if tenantname.Valid {
|
||||
builder.Field(61).(*array.StringBuilder).Append(tenantname.String)
|
||||
} else {
|
||||
builder.Field(61).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if tenantcontactno.Valid {
|
||||
builder.Field(62).(*array.StringBuilder).Append(tenantcontactno.String)
|
||||
} else {
|
||||
builder.Field(62).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if tenanttoken.Valid {
|
||||
builder.Field(63).(*array.StringBuilder).Append(tenanttoken.String)
|
||||
} else {
|
||||
builder.Field(63).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if tenantsuburb.Valid {
|
||||
builder.Field(64).(*array.StringBuilder).Append(tenantsuburb.String)
|
||||
} else {
|
||||
builder.Field(64).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if tenantcity.Valid {
|
||||
builder.Field(65).(*array.StringBuilder).Append(tenantcity.String)
|
||||
} else {
|
||||
builder.Field(65).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if tenantaddress.Valid {
|
||||
builder.Field(66).(*array.StringBuilder).Append(tenantaddress.String)
|
||||
} else {
|
||||
builder.Field(66).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if ridername.Valid {
|
||||
builder.Field(67).(*array.StringBuilder).Append(ridername.String)
|
||||
} else {
|
||||
builder.Field(67).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if userfcmtoken.Valid {
|
||||
builder.Field(68).(*array.StringBuilder).Append(userfcmtoken.String)
|
||||
} else {
|
||||
builder.Field(68).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if ridercontact.Valid {
|
||||
builder.Field(69).(*array.StringBuilder).Append(ridercontact.String)
|
||||
} else {
|
||||
builder.Field(69).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if locationname.Valid {
|
||||
builder.Field(70).(*array.StringBuilder).Append(locationname.String)
|
||||
} else {
|
||||
builder.Field(70).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if locationsuburb.Valid {
|
||||
builder.Field(71).(*array.StringBuilder).Append(locationsuburb.String)
|
||||
} else {
|
||||
builder.Field(71).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if locationcontactno.Valid {
|
||||
builder.Field(72).(*array.StringBuilder).Append(locationcontactno.String)
|
||||
} else {
|
||||
builder.Field(72).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if locationaddress.Valid {
|
||||
builder.Field(73).(*array.StringBuilder).Append(locationaddress.String)
|
||||
} else {
|
||||
builder.Field(73).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if slab.Valid {
|
||||
builder.Field(74).(*array.StringBuilder).Append(slab.String)
|
||||
} else {
|
||||
builder.Field(74).(*array.StringBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if pricingdate.Valid {
|
||||
builder.Field(75).(*array.TimestampBuilder).
|
||||
Append(arrow.Timestamp(pricingdate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(75).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
|
||||
if baseprice.Valid {
|
||||
builder.Field(76).(*array.Float64Builder).Append(baseprice.Float64)
|
||||
} else {
|
||||
builder.Field(76).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if minkm.Valid {
|
||||
builder.Field(77).(*array.Float64Builder).Append(minkm.Float64)
|
||||
} else {
|
||||
builder.Field(77).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if priceperkm.Valid {
|
||||
builder.Field(78).(*array.Float64Builder).Append(priceperkm.Float64)
|
||||
} else {
|
||||
builder.Field(78).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if maxkm.Valid {
|
||||
builder.Field(79).(*array.Float64Builder).Append(maxkm.Float64)
|
||||
} else {
|
||||
builder.Field(79).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if pricingorders.Valid {
|
||||
builder.Field(80).(*array.Int32Builder).Append(pricingorders.Int32)
|
||||
} else {
|
||||
builder.Field(80).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
|
||||
if othercharges.Valid {
|
||||
builder.Field(81).(*array.Float64Builder).Append(othercharges.Float64)
|
||||
} else {
|
||||
builder.Field(81).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
if surgecharges.Valid {
|
||||
builder.Field(82).(*array.Float64Builder).Append(surgecharges.Float64)
|
||||
} else {
|
||||
builder.Field(82).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
}
|
||||
record := builder.NewRecord()
|
||||
defer record.Release()
|
||||
|
||||
_ = os.MkdirAll("backups/parquet/deliveries", 0755)
|
||||
|
||||
filePath := "backups/parquet/deliveries/deliveries_" +
|
||||
time.Now().Format("2006_01_02") + ".parquet"
|
||||
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
parquetProps := parquet.NewWriterProperties(
|
||||
parquet.WithCompression(compress.Codecs.Snappy),
|
||||
)
|
||||
|
||||
writer, err := pqarrow.NewFileWriter(
|
||||
schema,
|
||||
file,
|
||||
parquetProps,
|
||||
pqarrow.ArrowWriterProperties{},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
if err := writer.Write(record); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filePath, nil
|
||||
}
|
||||
388
internal/backup/mysql_to_parquet_orders.go
Normal file
388
internal/backup/mysql_to_parquet_orders.go
Normal file
@@ -0,0 +1,388 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/apache/arrow/go/v14/arrow"
|
||||
"github.com/apache/arrow/go/v14/arrow/array"
|
||||
"github.com/apache/arrow/go/v14/arrow/memory"
|
||||
"github.com/apache/arrow/go/v14/parquet"
|
||||
"github.com/apache/arrow/go/v14/parquet/compress"
|
||||
"github.com/apache/arrow/go/v14/parquet/pqarrow"
|
||||
)
|
||||
|
||||
func BackupOrders(sqlDB *sql.DB) (string, error) {
|
||||
|
||||
rows, err := sqlDB.Query(`
|
||||
SELECT DISTINCT a.orderheaderid, a.applocationid,
|
||||
a.tenantid, a.locationid, a.partnerid, a.configid, a.categoryid, a.subcategoryid, a.moduleid,
|
||||
a.orderid, a.orderstatus, a.orderdate, a.ordernotes, a.itemcount, a.deliverytime AS deliverydate,
|
||||
a.pending, a.processing, a.ready, a.delivered AS completed, a.cancelled,
|
||||
a.deliverycharge, a.kms,
|
||||
a.customerid, a.pickupaddress, a.pickuplat, a.pickuplong,
|
||||
a.pickupcustomer, a.pickupcontactno, a.pickuplocation as pickupsuburb, a.pickupcity,
|
||||
a.deliveryid AS deliverycustomerid, a.deliveryaddress, a.deliverylat, a.deliverylong, a.deliverytype,
|
||||
a.deliverycustomer, a.deliverycontactno, a.deliverylocation as deliverysuburb, a.deliverycity, a.paymenttype, a.smsdelivery, a.orderamount, a.quantity, a.collectionamt,
|
||||
b.tenantname, b.tenanttoken, b.primarycontact AS tenantcontactno, b.postcode AS tenantpostcode, b.suburb AS tenantsuburb, b.city AS tenantcity, b.address AS tenantaddress,
|
||||
c.locationname, c.contactno AS locationcontactno, c.postcode AS locationpostcode, c.suburb AS locationsuburb, c.city AS locationcity, c.address AS locationaddress,
|
||||
d.locationname AS applocation, f.slab, f.pricingdate, f.baseprice, f.minkm, f.priceperkm, f.maxkm, f.orders, f.othercharges, f.surgecharges
|
||||
FROM orders a
|
||||
INNER JOIN tenants b ON a.tenantid = b.tenantid
|
||||
INNER JOIN tenantlocations c ON a.locationid = c.locationid
|
||||
INNER JOIN app_location d ON a.applocationid = d.applocationid
|
||||
INNER JOIN app_locationconfig e ON d.applocationid = e.applocationid
|
||||
LEFT JOIN tenantpricing f ON a.tenantid = f.tenantid
|
||||
WHERE a.orderdate >= NOW() - INTERVAL '3 months'
|
||||
`)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// ---------- Arrow Schema (aligned with SQL SELECT — 67 columns) ----------
|
||||
schema := arrow.NewSchema([]arrow.Field{
|
||||
// core ids
|
||||
{Name: "orderheaderid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "applocationid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "tenantid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "locationid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "partnerid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "configid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "categoryid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "subcategoryid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "moduleid", Type: arrow.PrimitiveTypes.Int32},
|
||||
// order meta
|
||||
{Name: "orderid", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "orderstatus", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "orderdate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
{Name: "ordernotes", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "itemcount", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "deliverydate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
// lifecycle
|
||||
{Name: "pending", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "processing", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "ready", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "completed", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "cancelled", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// charges & distance
|
||||
{Name: "deliverycharge", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "kms", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// pickup
|
||||
{Name: "customerid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "pickupaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplat", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplong", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickupcustomer", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickupcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickuplocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pickupcity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// delivery
|
||||
{Name: "deliverycustomerid", Type: arrow.PrimitiveTypes.Int32},
|
||||
{Name: "deliveryaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylat", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylong", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverytype", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverycustomer", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverycontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverylocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "deliverycity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// payment
|
||||
{Name: "paymenttype", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "smsdelivery", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "orderamount", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "quantity", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "collectionamt", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
// tenant
|
||||
{Name: "tenantname", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenanttoken", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantpostcode", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantsuburb", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantcity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "tenantaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// location
|
||||
{Name: "locationname", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationcontactno", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationpostcode", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationsuburb", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationcity", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "locationaddress", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
// app location + pricing
|
||||
{Name: "applocation", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "slab", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "pricingdate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true},
|
||||
{Name: "baseprice", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "minkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "priceperkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "maxkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "pricingorders", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "othercharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
{Name: "surgecharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
|
||||
}, nil)
|
||||
|
||||
pool := memory.NewGoAllocator()
|
||||
builder := array.NewRecordBuilder(pool, schema)
|
||||
defer builder.Release()
|
||||
|
||||
// ---------- Scan & Append ----------
|
||||
for rows.Next() {
|
||||
|
||||
var (
|
||||
orderheaderid, applocationid, tenantid, locationid, partnerid int32
|
||||
configid, categoryid, subcategoryid, moduleid int32
|
||||
|
||||
orderid, orderstatus, ordernotes sql.NullString
|
||||
pending, processing, ready, completed, cancelled sql.NullString
|
||||
pickupaddress, pickuplat, pickuplong sql.NullString
|
||||
pickupcustomer, pickupcontactno, pickupsuburb, pickupcity sql.NullString
|
||||
deliveryaddress, deliverylat, deliverylong sql.NullString
|
||||
deliverytype, deliverycustomer, deliverycontactno sql.NullString
|
||||
deliverysuburb, deliverycity sql.NullString
|
||||
kms sql.NullString
|
||||
|
||||
itemcount, paymenttype, smsdelivery, quantity sql.NullInt32
|
||||
customerid, deliverycustomerid int32
|
||||
orderamount, deliverycharge, collectionamt sql.NullFloat64
|
||||
|
||||
orderdate, deliverydate sql.NullTime
|
||||
|
||||
tenantname, tenanttoken, tenantcontactno sql.NullString
|
||||
tenantpostcode, tenantsuburb, tenantcity, tenantaddress sql.NullString
|
||||
|
||||
locationname, locationcontactno, locationpostcode sql.NullString
|
||||
locationsuburb, locationcity, locationaddress sql.NullString
|
||||
|
||||
applocation sql.NullString
|
||||
slab sql.NullString
|
||||
pricingdate sql.NullTime
|
||||
baseprice, minkm, priceperkm, maxkm sql.NullFloat64
|
||||
pricingorders sql.NullInt32
|
||||
othercharges, surgecharges sql.NullFloat64
|
||||
)
|
||||
|
||||
if err := rows.Scan(
|
||||
// core ids
|
||||
&orderheaderid, &applocationid,
|
||||
&tenantid, &locationid, &partnerid, &configid, &categoryid, &subcategoryid, &moduleid,
|
||||
// order meta
|
||||
&orderid, &orderstatus, &orderdate, &ordernotes, &itemcount, &deliverydate,
|
||||
// lifecycle
|
||||
&pending, &processing, &ready, &completed, &cancelled,
|
||||
// charges & distance
|
||||
&deliverycharge, &kms,
|
||||
// pickup
|
||||
&customerid, &pickupaddress, &pickuplat, &pickuplong,
|
||||
&pickupcustomer, &pickupcontactno, &pickupsuburb, &pickupcity,
|
||||
// delivery
|
||||
&deliverycustomerid, &deliveryaddress, &deliverylat, &deliverylong,
|
||||
&deliverytype, &deliverycustomer, &deliverycontactno,
|
||||
&deliverysuburb, &deliverycity,
|
||||
// payment
|
||||
&paymenttype, &smsdelivery, &orderamount, &quantity, &collectionamt,
|
||||
// tenant
|
||||
&tenantname, &tenanttoken, &tenantcontactno,
|
||||
&tenantpostcode, &tenantsuburb, &tenantcity, &tenantaddress,
|
||||
// location
|
||||
&locationname, &locationcontactno, &locationpostcode,
|
||||
&locationsuburb, &locationcity, &locationaddress,
|
||||
// app location + pricing
|
||||
&applocation,
|
||||
&slab, &pricingdate, &baseprice, &minkm, &priceperkm,
|
||||
&maxkm, &pricingorders, &othercharges, &surgecharges,
|
||||
); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// core ids (not null)
|
||||
builder.Field(0).(*array.Int32Builder).Append(orderheaderid)
|
||||
builder.Field(1).(*array.Int32Builder).Append(applocationid)
|
||||
builder.Field(2).(*array.Int32Builder).Append(tenantid)
|
||||
builder.Field(3).(*array.Int32Builder).Append(locationid)
|
||||
builder.Field(4).(*array.Int32Builder).Append(partnerid)
|
||||
builder.Field(5).(*array.Int32Builder).Append(configid)
|
||||
builder.Field(6).(*array.Int32Builder).Append(categoryid)
|
||||
builder.Field(7).(*array.Int32Builder).Append(subcategoryid)
|
||||
builder.Field(8).(*array.Int32Builder).Append(moduleid)
|
||||
|
||||
// order meta
|
||||
appendString(builder.Field(9), orderid)
|
||||
appendString(builder.Field(10), orderstatus)
|
||||
if orderdate.Valid {
|
||||
builder.Field(11).(*array.TimestampBuilder).Append(arrow.Timestamp(orderdate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(11).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
appendString(builder.Field(12), ordernotes)
|
||||
if itemcount.Valid {
|
||||
builder.Field(13).(*array.Int32Builder).Append(itemcount.Int32)
|
||||
} else {
|
||||
builder.Field(13).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if deliverydate.Valid {
|
||||
builder.Field(14).(*array.TimestampBuilder).Append(arrow.Timestamp(deliverydate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(14).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
|
||||
// lifecycle
|
||||
appendString(builder.Field(15), pending)
|
||||
appendString(builder.Field(16), processing)
|
||||
appendString(builder.Field(17), ready)
|
||||
appendString(builder.Field(18), completed)
|
||||
appendString(builder.Field(19), cancelled)
|
||||
|
||||
// charges & distance
|
||||
if deliverycharge.Valid {
|
||||
builder.Field(20).(*array.Float64Builder).Append(deliverycharge.Float64)
|
||||
} else {
|
||||
builder.Field(20).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
appendString(builder.Field(21), kms)
|
||||
|
||||
// pickup
|
||||
builder.Field(22).(*array.Int32Builder).Append(customerid)
|
||||
appendString(builder.Field(23), pickupaddress)
|
||||
appendString(builder.Field(24), pickuplat)
|
||||
appendString(builder.Field(25), pickuplong)
|
||||
appendString(builder.Field(26), pickupcustomer)
|
||||
appendString(builder.Field(27), pickupcontactno)
|
||||
appendString(builder.Field(28), pickupsuburb)
|
||||
appendString(builder.Field(29), pickupcity)
|
||||
|
||||
// delivery
|
||||
builder.Field(30).(*array.Int32Builder).Append(deliverycustomerid)
|
||||
appendString(builder.Field(31), deliveryaddress)
|
||||
appendString(builder.Field(32), deliverylat)
|
||||
appendString(builder.Field(33), deliverylong)
|
||||
appendString(builder.Field(34), deliverytype)
|
||||
appendString(builder.Field(35), deliverycustomer)
|
||||
appendString(builder.Field(36), deliverycontactno)
|
||||
appendString(builder.Field(37), deliverysuburb)
|
||||
appendString(builder.Field(38), deliverycity)
|
||||
|
||||
// payment
|
||||
if paymenttype.Valid {
|
||||
builder.Field(39).(*array.Int32Builder).Append(paymenttype.Int32)
|
||||
} else {
|
||||
builder.Field(39).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if smsdelivery.Valid {
|
||||
builder.Field(40).(*array.Int32Builder).Append(smsdelivery.Int32)
|
||||
} else {
|
||||
builder.Field(40).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if orderamount.Valid {
|
||||
builder.Field(41).(*array.Float64Builder).Append(orderamount.Float64)
|
||||
} else {
|
||||
builder.Field(41).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if quantity.Valid {
|
||||
builder.Field(42).(*array.Int32Builder).Append(quantity.Int32)
|
||||
} else {
|
||||
builder.Field(42).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if collectionamt.Valid {
|
||||
builder.Field(43).(*array.Float64Builder).Append(collectionamt.Float64)
|
||||
} else {
|
||||
builder.Field(43).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
|
||||
// tenant
|
||||
appendString(builder.Field(44), tenantname)
|
||||
appendString(builder.Field(45), tenanttoken)
|
||||
appendString(builder.Field(46), tenantcontactno)
|
||||
appendString(builder.Field(47), tenantpostcode)
|
||||
appendString(builder.Field(48), tenantsuburb)
|
||||
appendString(builder.Field(49), tenantcity)
|
||||
appendString(builder.Field(50), tenantaddress)
|
||||
|
||||
// location
|
||||
appendString(builder.Field(51), locationname)
|
||||
appendString(builder.Field(52), locationcontactno)
|
||||
appendString(builder.Field(53), locationpostcode)
|
||||
appendString(builder.Field(54), locationsuburb)
|
||||
appendString(builder.Field(55), locationcity)
|
||||
appendString(builder.Field(56), locationaddress)
|
||||
|
||||
// app location + pricing
|
||||
appendString(builder.Field(57), applocation)
|
||||
appendString(builder.Field(58), slab)
|
||||
if pricingdate.Valid {
|
||||
builder.Field(59).(*array.TimestampBuilder).Append(arrow.Timestamp(pricingdate.Time.UnixMilli()))
|
||||
} else {
|
||||
builder.Field(59).(*array.TimestampBuilder).AppendNull()
|
||||
}
|
||||
if baseprice.Valid {
|
||||
builder.Field(60).(*array.Float64Builder).Append(baseprice.Float64)
|
||||
} else {
|
||||
builder.Field(60).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if minkm.Valid {
|
||||
builder.Field(61).(*array.Float64Builder).Append(minkm.Float64)
|
||||
} else {
|
||||
builder.Field(61).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if priceperkm.Valid {
|
||||
builder.Field(62).(*array.Float64Builder).Append(priceperkm.Float64)
|
||||
} else {
|
||||
builder.Field(62).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if maxkm.Valid {
|
||||
builder.Field(63).(*array.Float64Builder).Append(maxkm.Float64)
|
||||
} else {
|
||||
builder.Field(63).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if pricingorders.Valid {
|
||||
builder.Field(64).(*array.Int32Builder).Append(pricingorders.Int32)
|
||||
} else {
|
||||
builder.Field(64).(*array.Int32Builder).AppendNull()
|
||||
}
|
||||
if othercharges.Valid {
|
||||
builder.Field(65).(*array.Float64Builder).Append(othercharges.Float64)
|
||||
} else {
|
||||
builder.Field(65).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
if surgecharges.Valid {
|
||||
builder.Field(66).(*array.Float64Builder).Append(surgecharges.Float64)
|
||||
} else {
|
||||
builder.Field(66).(*array.Float64Builder).AppendNull()
|
||||
}
|
||||
}
|
||||
|
||||
record := builder.NewRecord()
|
||||
defer record.Release()
|
||||
|
||||
_ = os.MkdirAll("backups/parquet/orders", 0755)
|
||||
|
||||
filePath := "backups/parquet/orders/orders_last_3_months_" +
|
||||
time.Now().Format("2006_01_02") + ".parquet"
|
||||
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
parquetProps := parquet.NewWriterProperties(
|
||||
parquet.WithCompression(compress.Codecs.Snappy),
|
||||
)
|
||||
|
||||
writer, err := pqarrow.NewFileWriter(
|
||||
schema,
|
||||
file,
|
||||
parquetProps,
|
||||
pqarrow.ArrowWriterProperties{},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
if err := writer.Write(record); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filePath, nil
|
||||
}
|
||||
67
internal/backup/s3_upload.go
Normal file
67
internal/backup/s3_upload.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
func UploadParquetToSpaces(localFilePath, objectPath string) (string, error) {
|
||||
|
||||
region := os.Getenv("DO_SPACES_REGION")
|
||||
endpoint := os.Getenv("DO_SPACES_ENDPOINT")
|
||||
bucket := os.Getenv("DO_SPACES_BUCKET")
|
||||
accessKey := os.Getenv("DO_SPACES_ACCESS_KEY")
|
||||
secretKey := os.Getenv("DO_SPACES_SECRET_KEY")
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(
|
||||
context.TODO(),
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
||||
),
|
||||
config.WithEndpointResolver(
|
||||
aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
URL: "https://" + endpoint,
|
||||
SigningRegion: region,
|
||||
}, nil
|
||||
}),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
client := s3.NewFromConfig(cfg)
|
||||
|
||||
file, err := os.Open(localFilePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(objectPath),
|
||||
Body: file,
|
||||
ContentType: aws.String("application/octet-stream"),
|
||||
ACL: "public-read",
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
cdnURL := fmt.Sprintf(
|
||||
"https://images.nearle.app/%s",
|
||||
filepath.ToSlash(objectPath),
|
||||
)
|
||||
|
||||
return cdnURL, nil
|
||||
}
|
||||
Reference in New Issue
Block a user