package backup import ( "database/sql" "os" "time" "github.com/apache/arrow/go/v14/arrow" "github.com/apache/arrow/go/v14/arrow/array" "github.com/apache/arrow/go/v14/arrow/memory" "github.com/apache/arrow/go/v14/parquet" "github.com/apache/arrow/go/v14/parquet/compress" "github.com/apache/arrow/go/v14/parquet/pqarrow" ) func BackupOrders(sqlDB *sql.DB) (string, error) { rows, err := sqlDB.Query(` SELECT DISTINCT a.orderheaderid, a.applocationid, a.tenantid, a.locationid, a.partnerid, a.configid, a.categoryid, a.subcategoryid, a.moduleid, a.orderid, a.orderstatus, a.orderdate, a.ordernotes, a.itemcount, a.deliverytime AS deliverydate, a.pending, a.processing, a.ready, a.delivered AS completed, a.cancelled, a.deliverycharge, a.kms, a.customerid, a.pickupaddress, a.pickuplat, a.pickuplong, a.pickupcustomer, a.pickupcontactno, a.pickuplocation as pickupsuburb, a.pickupcity, a.deliveryid AS deliverycustomerid, a.deliveryaddress, a.deliverylat, a.deliverylong, a.deliverytype, a.deliverycustomer, a.deliverycontactno, a.deliverylocation as deliverysuburb, a.deliverycity, a.paymenttype, a.smsdelivery, a.orderamount, a.quantity, a.collectionamt, b.tenantname, b.tenanttoken, b.primarycontact AS tenantcontactno, b.postcode AS tenantpostcode, b.suburb AS tenantsuburb, b.city AS tenantcity, b.address AS tenantaddress, c.locationname, c.contactno AS locationcontactno, c.postcode AS locationpostcode, c.suburb AS locationsuburb, c.city AS locationcity, c.address AS locationaddress, d.locationname AS applocation, f.slab, f.pricingdate, f.baseprice, f.minkm, f.priceperkm, f.maxkm, f.orders, f.othercharges, f.surgecharges FROM orders a INNER JOIN tenants b ON a.tenantid = b.tenantid INNER JOIN tenantlocations c ON a.locationid = c.locationid INNER JOIN app_location d ON a.applocationid = d.applocationid INNER JOIN app_locationconfig e ON d.applocationid = e.applocationid LEFT JOIN tenantpricing f ON a.tenantid = f.tenantid WHERE a.orderdate >= NOW() - INTERVAL '3 months' `) if err != nil { return "", err } defer rows.Close() // ---------- Arrow Schema (aligned with SQL SELECT — 67 columns) ---------- schema := arrow.NewSchema([]arrow.Field{ // core ids {Name: "orderheaderid", Type: arrow.PrimitiveTypes.Int32}, {Name: "applocationid", Type: arrow.PrimitiveTypes.Int32}, {Name: "tenantid", Type: arrow.PrimitiveTypes.Int32}, {Name: "locationid", Type: arrow.PrimitiveTypes.Int32}, {Name: "partnerid", Type: arrow.PrimitiveTypes.Int32}, {Name: "configid", Type: arrow.PrimitiveTypes.Int32}, {Name: "categoryid", Type: arrow.PrimitiveTypes.Int32}, {Name: "subcategoryid", Type: arrow.PrimitiveTypes.Int32}, {Name: "moduleid", Type: arrow.PrimitiveTypes.Int32}, // order meta {Name: "orderid", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "orderstatus", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "orderdate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true}, {Name: "ordernotes", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "itemcount", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, {Name: "deliverydate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true}, // lifecycle {Name: "pending", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "processing", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "ready", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "completed", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "cancelled", Type: arrow.BinaryTypes.String, Nullable: true}, // charges & distance {Name: "deliverycharge", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, {Name: "kms", Type: arrow.BinaryTypes.String, Nullable: true}, // pickup {Name: "customerid", Type: arrow.PrimitiveTypes.Int32}, {Name: "pickupaddress", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "pickuplat", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "pickuplong", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "pickupcustomer", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "pickupcontactno", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "pickuplocation", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "pickupcity", Type: arrow.BinaryTypes.String, Nullable: true}, // delivery {Name: "deliverycustomerid", Type: arrow.PrimitiveTypes.Int32}, {Name: "deliveryaddress", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "deliverylat", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "deliverylong", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "deliverytype", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "deliverycustomer", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "deliverycontactno", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "deliverylocation", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "deliverycity", Type: arrow.BinaryTypes.String, Nullable: true}, // payment {Name: "paymenttype", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, {Name: "smsdelivery", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, {Name: "orderamount", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, {Name: "quantity", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, {Name: "collectionamt", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, // tenant {Name: "tenantname", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "tenanttoken", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "tenantcontactno", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "tenantpostcode", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "tenantsuburb", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "tenantcity", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "tenantaddress", Type: arrow.BinaryTypes.String, Nullable: true}, // location {Name: "locationname", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "locationcontactno", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "locationpostcode", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "locationsuburb", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "locationcity", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "locationaddress", Type: arrow.BinaryTypes.String, Nullable: true}, // app location + pricing {Name: "applocation", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "slab", Type: arrow.BinaryTypes.String, Nullable: true}, {Name: "pricingdate", Type: arrow.FixedWidthTypes.Timestamp_ms, Nullable: true}, {Name: "baseprice", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, {Name: "minkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, {Name: "priceperkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, {Name: "maxkm", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, {Name: "pricingorders", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, {Name: "othercharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, {Name: "surgecharges", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, }, nil) pool := memory.NewGoAllocator() builder := array.NewRecordBuilder(pool, schema) defer builder.Release() // ---------- Scan & Append ---------- for rows.Next() { var ( orderheaderid, applocationid, tenantid, locationid, partnerid int32 configid, categoryid, subcategoryid, moduleid int32 orderid, orderstatus, ordernotes sql.NullString pending, processing, ready, completed, cancelled sql.NullString pickupaddress, pickuplat, pickuplong sql.NullString pickupcustomer, pickupcontactno, pickupsuburb, pickupcity sql.NullString deliveryaddress, deliverylat, deliverylong sql.NullString deliverytype, deliverycustomer, deliverycontactno sql.NullString deliverysuburb, deliverycity sql.NullString kms sql.NullString itemcount, paymenttype, smsdelivery, quantity sql.NullInt32 customerid, deliverycustomerid int32 orderamount, deliverycharge, collectionamt sql.NullFloat64 orderdate, deliverydate sql.NullTime tenantname, tenanttoken, tenantcontactno sql.NullString tenantpostcode, tenantsuburb, tenantcity, tenantaddress sql.NullString locationname, locationcontactno, locationpostcode sql.NullString locationsuburb, locationcity, locationaddress sql.NullString applocation sql.NullString slab sql.NullString pricingdate sql.NullTime baseprice, minkm, priceperkm, maxkm sql.NullFloat64 pricingorders sql.NullInt32 othercharges, surgecharges sql.NullFloat64 ) if err := rows.Scan( // core ids &orderheaderid, &applocationid, &tenantid, &locationid, &partnerid, &configid, &categoryid, &subcategoryid, &moduleid, // order meta &orderid, &orderstatus, &orderdate, &ordernotes, &itemcount, &deliverydate, // lifecycle &pending, &processing, &ready, &completed, &cancelled, // charges & distance &deliverycharge, &kms, // pickup &customerid, &pickupaddress, &pickuplat, &pickuplong, &pickupcustomer, &pickupcontactno, &pickupsuburb, &pickupcity, // delivery &deliverycustomerid, &deliveryaddress, &deliverylat, &deliverylong, &deliverytype, &deliverycustomer, &deliverycontactno, &deliverysuburb, &deliverycity, // payment &paymenttype, &smsdelivery, &orderamount, &quantity, &collectionamt, // tenant &tenantname, &tenanttoken, &tenantcontactno, &tenantpostcode, &tenantsuburb, &tenantcity, &tenantaddress, // location &locationname, &locationcontactno, &locationpostcode, &locationsuburb, &locationcity, &locationaddress, // app location + pricing &applocation, &slab, &pricingdate, &baseprice, &minkm, &priceperkm, &maxkm, &pricingorders, &othercharges, &surgecharges, ); err != nil { return "", err } // core ids (not null) builder.Field(0).(*array.Int32Builder).Append(orderheaderid) builder.Field(1).(*array.Int32Builder).Append(applocationid) builder.Field(2).(*array.Int32Builder).Append(tenantid) builder.Field(3).(*array.Int32Builder).Append(locationid) builder.Field(4).(*array.Int32Builder).Append(partnerid) builder.Field(5).(*array.Int32Builder).Append(configid) builder.Field(6).(*array.Int32Builder).Append(categoryid) builder.Field(7).(*array.Int32Builder).Append(subcategoryid) builder.Field(8).(*array.Int32Builder).Append(moduleid) // order meta appendString(builder.Field(9), orderid) appendString(builder.Field(10), orderstatus) if orderdate.Valid { builder.Field(11).(*array.TimestampBuilder).Append(arrow.Timestamp(orderdate.Time.UnixMilli())) } else { builder.Field(11).(*array.TimestampBuilder).AppendNull() } appendString(builder.Field(12), ordernotes) if itemcount.Valid { builder.Field(13).(*array.Int32Builder).Append(itemcount.Int32) } else { builder.Field(13).(*array.Int32Builder).AppendNull() } if deliverydate.Valid { builder.Field(14).(*array.TimestampBuilder).Append(arrow.Timestamp(deliverydate.Time.UnixMilli())) } else { builder.Field(14).(*array.TimestampBuilder).AppendNull() } // lifecycle appendString(builder.Field(15), pending) appendString(builder.Field(16), processing) appendString(builder.Field(17), ready) appendString(builder.Field(18), completed) appendString(builder.Field(19), cancelled) // charges & distance if deliverycharge.Valid { builder.Field(20).(*array.Float64Builder).Append(deliverycharge.Float64) } else { builder.Field(20).(*array.Float64Builder).AppendNull() } appendString(builder.Field(21), kms) // pickup builder.Field(22).(*array.Int32Builder).Append(customerid) appendString(builder.Field(23), pickupaddress) appendString(builder.Field(24), pickuplat) appendString(builder.Field(25), pickuplong) appendString(builder.Field(26), pickupcustomer) appendString(builder.Field(27), pickupcontactno) appendString(builder.Field(28), pickupsuburb) appendString(builder.Field(29), pickupcity) // delivery builder.Field(30).(*array.Int32Builder).Append(deliverycustomerid) appendString(builder.Field(31), deliveryaddress) appendString(builder.Field(32), deliverylat) appendString(builder.Field(33), deliverylong) appendString(builder.Field(34), deliverytype) appendString(builder.Field(35), deliverycustomer) appendString(builder.Field(36), deliverycontactno) appendString(builder.Field(37), deliverysuburb) appendString(builder.Field(38), deliverycity) // payment if paymenttype.Valid { builder.Field(39).(*array.Int32Builder).Append(paymenttype.Int32) } else { builder.Field(39).(*array.Int32Builder).AppendNull() } if smsdelivery.Valid { builder.Field(40).(*array.Int32Builder).Append(smsdelivery.Int32) } else { builder.Field(40).(*array.Int32Builder).AppendNull() } if orderamount.Valid { builder.Field(41).(*array.Float64Builder).Append(orderamount.Float64) } else { builder.Field(41).(*array.Float64Builder).AppendNull() } if quantity.Valid { builder.Field(42).(*array.Int32Builder).Append(quantity.Int32) } else { builder.Field(42).(*array.Int32Builder).AppendNull() } if collectionamt.Valid { builder.Field(43).(*array.Float64Builder).Append(collectionamt.Float64) } else { builder.Field(43).(*array.Float64Builder).AppendNull() } // tenant appendString(builder.Field(44), tenantname) appendString(builder.Field(45), tenanttoken) appendString(builder.Field(46), tenantcontactno) appendString(builder.Field(47), tenantpostcode) appendString(builder.Field(48), tenantsuburb) appendString(builder.Field(49), tenantcity) appendString(builder.Field(50), tenantaddress) // location appendString(builder.Field(51), locationname) appendString(builder.Field(52), locationcontactno) appendString(builder.Field(53), locationpostcode) appendString(builder.Field(54), locationsuburb) appendString(builder.Field(55), locationcity) appendString(builder.Field(56), locationaddress) // app location + pricing appendString(builder.Field(57), applocation) appendString(builder.Field(58), slab) if pricingdate.Valid { builder.Field(59).(*array.TimestampBuilder).Append(arrow.Timestamp(pricingdate.Time.UnixMilli())) } else { builder.Field(59).(*array.TimestampBuilder).AppendNull() } if baseprice.Valid { builder.Field(60).(*array.Float64Builder).Append(baseprice.Float64) } else { builder.Field(60).(*array.Float64Builder).AppendNull() } if minkm.Valid { builder.Field(61).(*array.Float64Builder).Append(minkm.Float64) } else { builder.Field(61).(*array.Float64Builder).AppendNull() } if priceperkm.Valid { builder.Field(62).(*array.Float64Builder).Append(priceperkm.Float64) } else { builder.Field(62).(*array.Float64Builder).AppendNull() } if maxkm.Valid { builder.Field(63).(*array.Float64Builder).Append(maxkm.Float64) } else { builder.Field(63).(*array.Float64Builder).AppendNull() } if pricingorders.Valid { builder.Field(64).(*array.Int32Builder).Append(pricingorders.Int32) } else { builder.Field(64).(*array.Int32Builder).AppendNull() } if othercharges.Valid { builder.Field(65).(*array.Float64Builder).Append(othercharges.Float64) } else { builder.Field(65).(*array.Float64Builder).AppendNull() } if surgecharges.Valid { builder.Field(66).(*array.Float64Builder).Append(surgecharges.Float64) } else { builder.Field(66).(*array.Float64Builder).AppendNull() } } record := builder.NewRecord() defer record.Release() _ = os.MkdirAll("backups/parquet/orders", 0755) filePath := "backups/parquet/orders/orders_last_3_months_" + time.Now().Format("2006_01_02") + ".parquet" file, err := os.Create(filePath) if err != nil { return "", err } defer file.Close() parquetProps := parquet.NewWriterProperties( parquet.WithCompression(compress.Codecs.Snappy), ) writer, err := pqarrow.NewFileWriter( schema, file, parquetProps, pqarrow.ArrowWriterProperties{}, ) if err != nil { return "", err } defer writer.Close() if err := writer.Write(record); err != nil { return "", err } return filePath, nil }