{ "conditions": { "execution": { "elapsed_seconds": 2.8805396700045094, "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition, 595.91.07, 97887 MiB", "rustc": "rustc 1.98.1 (48a229cea 2026-09-01)", "server_binary_sha256": "bc6a0cc9cbc276c54c8d621622905f33cba754ebecaadeb897785ac519eeaa63", "status": "ok" }, "feature_input": { "description": "normalized byte and byte-entropy histograms from source features f7 through f518", "dimension": 512, "field": "histogram", "source_dimensions_zero_based": [ 7, 518 ] }, "held_out_test_period": { "split": "test", "week_ids_in_supervised_report": [ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 ] }, "label_semantics": { "0": "benign", "1": "malicious" }, "models": { "logistic_regression": { "max_iter": 1000, "output": "prediction is a zero-based class ID, not a score probability", "penalty_l2": 1.0 }, "official_pretrained_reference": { "comparison_note": "reference result only, not a fair algorithm ranking", "feature_count": 2568, "model": "EMBER2024_all.model", "threshold": 0.5, "training_corpus": "larger upstream training corpus; not fitted on the 15,588-row histogram cohort" }, "random_forest_classifier": { "max_depth": 12, "n_trees": 100, "output": "prediction is a zero-based class ID, not a score probability", "seed": 42 } }, "train_test_sampling": { "rule": "retain unique SHA256 rows when integer(first 8 hexadecimal SHA256 characters, base 16) modulo 100 equals 0", "selection_uses_label_or_family": false, "test": { "metadata_path": "target/ember2024-research/input_metadata_test.parquet", "selected_labels": { "benign": 1831, "malicious": 1787 }, "selected_win32_test_rows": 3618, "unique_win32_metadata_rows": 359994, "vectors_path": "target/ember2024-research/input_research_vectors_test.parquet" }, "train": { "metadata_path": "target/ember2024-research/input_metadata_train.parquet", "selected_labels": { "benign": 7737, "malicious": 7851 }, "selected_win32_training_rows": 15588, "unique_win32_metadata_rows": 1560000, "vectors_path": "target/ember2024-research/input_research_vectors_train.parquet" } } }, "dataset": "EMBER2024", "name": "ember2024_temporal_model_comparison", "result": { "csv_export_status": "ok", "data": [ { "false_negative": 394, "false_positive": 241, "files": 3618, "model": "logistic", "true_negative": 1590, "true_positive": 1393 }, { "false_negative": 270, "false_positive": 112, "files": 3618, "model": "random_forest", "true_negative": 1719, "true_positive": 1517 } ], "elapsed_s": 2.8805396700045094, "query_id": "fada4867-5ff5-4e0f-9d3a-6fbf7aea06f5", "rows": 2, "schema": "model: string not null\nfiles: int64 not null\ntrue_positive: int64\nfalse_positive: int64\nfalse_negative: int64\ntrue_negative: int64", "source_name": "16_temporal_model_comparison", "sql": "WITH logistic AS (\nSELECT p.id, p.prediction, m.label, m.week_id, m.sha256, m.family\nFROM cuml_logistic_regression(\n training => (\n SELECT v.id, v.histogram AS vector, v.label\n FROM ember_research_vectors_train v\n JOIN ember_metadata_train m ON m.id = v.id\n WHERE m.file_type = 'Win32'\n ORDER BY v.id\n ),\n predict => (\n SELECT v.id, v.histogram AS vector\n FROM ember_research_vectors_test v\n JOIN ember_metadata_test m ON m.id = v.id\n WHERE m.file_type = 'Win32'\n ORDER BY v.id\n ),\n max_iter => 1000,\n penalty_l2 => 1.0\n) p\nJOIN ember_metadata_test m ON m.id = p.id\n),\nrandom_forest AS (\nSELECT p.id, p.prediction, m.label, m.week_id, m.sha256, m.family\nFROM cuml_random_forest_classifier(\n training => (\n SELECT v.id, v.histogram AS vector, v.label\n FROM ember_research_vectors_train v\n JOIN ember_metadata_train m ON m.id = v.id\n WHERE m.file_type = 'Win32'\n ORDER BY v.id\n ),\n predict => (\n SELECT v.id, v.histogram AS vector\n FROM ember_research_vectors_test v\n JOIN ember_metadata_test m ON m.id = v.id\n WHERE m.file_type = 'Win32'\n ORDER BY v.id\n ),\n n_trees => 100,\n max_depth => 12,\n seed => 42\n) p\nJOIN ember_metadata_test m ON m.id = p.id\n),\nclassified AS (\n SELECT 'logistic' AS model, id, prediction, label FROM logistic\n UNION ALL\n SELECT 'random_forest' AS model, id, prediction, label FROM random_forest\n)\nSELECT model, COUNT(*) AS files,\n SUM(CASE WHEN prediction = 1 AND label = 1 THEN 1 ELSE 0 END) AS true_positive,\n SUM(CASE WHEN prediction = 1 AND label = 0 THEN 1 ELSE 0 END) AS false_positive,\n SUM(CASE WHEN prediction = 0 AND label = 1 THEN 1 ELSE 0 END) AS false_negative,\n SUM(CASE WHEN prediction = 0 AND label = 0 THEN 1 ELSE 0 END) AS true_negative\nFROM classified\nGROUP BY model\nORDER BY model;\n", "status": "ok" }, "source_query_ids": { "13_temporal_logistic": "56061a8b-5c0b-458b-80f3-31977b9352da", "14_temporal_random_forest": "80ab6789-7385-4af0-9a5d-f6c2f177143b", "15_forest_temporal_test_official_model": "3c3ae508-178f-4f15-9ab5-c483ba987231", "16_temporal_model_comparison": "fada4867-5ff5-4e0f-9d3a-6fbf7aea06f5" }, "supervised_report_metrics": { "histogram_classifiers": { "13_temporal_logistic": { "by_week_id": { "52": { "accuracy": 0.8445229681978799, "confusion_matrix": { "fn": 28, "fp": 16, "tn": 138, "tp": 101 }, "f1": 0.8211382113821137, "false_positive_rate": 0.1038961038961039, "false_positive_rate_wilson_95": { "lower": 0.06496887857583979, "upper": 0.16210364779639536 }, "negative_rows": 154, "positive_rows": 129, "precision": 0.8632478632478633, "recall": 0.7829457364341085, "rows": 283 }, "53": { "accuracy": 0.8672566371681416, "confusion_matrix": { "fn": 31, "fp": 14, "tn": 164, "tp": 130 }, "f1": 0.8524590163934427, "false_positive_rate": 0.07865168539325842, "false_positive_rate_wilson_95": { "lower": 0.04742689159231053, "upper": 0.12767871428655334 }, "negative_rows": 178, "positive_rows": 161, "precision": 0.9027777777777778, "recall": 0.8074534161490683, "rows": 339 }, "54": { "accuracy": 0.8366666666666667, "confusion_matrix": { "fn": 37, "fp": 12, "tn": 140, "tp": 111 }, "f1": 0.8191881918819189, "false_positive_rate": 0.07894736842105263, "false_positive_rate_wilson_95": { "lower": 0.045735835972808184, "upper": 0.13291661653488784 }, "negative_rows": 152, "positive_rows": 148, "precision": 0.9024390243902439, "recall": 0.75, "rows": 300 }, "55": { "accuracy": 0.8518518518518519, "confusion_matrix": { "fn": 28, "fp": 16, "tn": 124, "tp": 129 }, "f1": 0.8543046357615894, "false_positive_rate": 0.11428571428571428, "false_positive_rate_wilson_95": { "lower": 0.07158260814437095, "upper": 0.17759074642631906 }, "negative_rows": 140, "positive_rows": 157, "precision": 0.8896551724137931, "recall": 0.821656050955414, "rows": 297 }, "56": { "accuracy": 0.8264984227129337, "confusion_matrix": { "fn": 34, "fp": 21, "tn": 127, "tp": 135 }, "f1": 0.8307692307692308, "false_positive_rate": 0.14189189189189189, "false_positive_rate_wilson_95": { "lower": 0.0947158924766123, "upper": 0.20718754797193592 }, "negative_rows": 148, "positive_rows": 169, "precision": 0.8653846153846154, "recall": 0.7988165680473372, "rows": 317 }, "57": { "accuracy": 0.852112676056338, "confusion_matrix": { "fn": 28, "fp": 14, "tn": 134, "tp": 108 }, "f1": 0.8372093023255813, "false_positive_rate": 0.0945945945945946, "false_positive_rate_wilson_95": { "lower": 0.057185755394348226, "upper": 0.15251625266061208 }, "negative_rows": 148, "positive_rows": 136, "precision": 0.8852459016393442, "recall": 0.7941176470588235, "rows": 284 }, "58": { "accuracy": 0.8517241379310345, "confusion_matrix": { "fn": 21, "fp": 22, "tn": 128, "tp": 119 }, "f1": 0.8469750889679715, "false_positive_rate": 0.14666666666666667, "false_positive_rate_wilson_95": { "lower": 0.0988943721258812, "upper": 0.2120846005608929 }, "negative_rows": 150, "positive_rows": 140, "precision": 0.8439716312056738, "recall": 0.85, "rows": 290 }, "59": { "accuracy": 0.83125, "confusion_matrix": { "fn": 31, "fp": 23, "tn": 146, "tp": 120 }, "f1": 0.8163265306122448, "false_positive_rate": 0.13609467455621302, "false_positive_rate_wilson_95": { "lower": 0.09242825369290503, "upper": 0.19593692883196423 }, "negative_rows": 169, "positive_rows": 151, "precision": 0.8391608391608392, "recall": 0.7947019867549668, "rows": 320 }, "60": { "accuracy": 0.7758007117437722, "confusion_matrix": { "fn": 40, "fp": 23, "tn": 125, "tp": 93 }, "f1": 0.7469879518072289, "false_positive_rate": 0.1554054054054054, "false_positive_rate_wilson_95": { "lower": 0.1058427333885073, "upper": 0.2224039734582089 }, "negative_rows": 148, "positive_rows": 133, "precision": 0.8017241379310345, "recall": 0.6992481203007519, "rows": 281 }, "61": { "accuracy": 0.7438271604938271, "confusion_matrix": { "fn": 53, "fp": 30, "tn": 124, "tp": 117 }, "f1": 0.7381703470031545, "false_positive_rate": 0.19480519480519481, "false_positive_rate_wilson_95": { "lower": 0.1400022499888041, "upper": 0.2644634670520984 }, "negative_rows": 154, "positive_rows": 170, "precision": 0.7959183673469388, "recall": 0.6882352941176471, "rows": 324 }, "62": { "accuracy": 0.7972508591065293, "confusion_matrix": { "fn": 34, "fp": 25, "tn": 124, "tp": 108 }, "f1": 0.7854545454545454, "false_positive_rate": 0.16778523489932887, "false_positive_rate_wilson_95": { "lower": 0.11630849695340292, "upper": 0.23596149075888526 }, "negative_rows": 149, "positive_rows": 142, "precision": 0.8120300751879699, "recall": 0.7605633802816901, "rows": 291 }, "63": { "accuracy": 0.815068493150685, "confusion_matrix": { "fn": 29, "fp": 25, "tn": 116, "tp": 122 }, "f1": 0.8187919463087249, "false_positive_rate": 0.1773049645390071, "false_positive_rate_wilson_95": { "lower": 0.1230786363817971, "upper": 0.24864821074521876 }, "negative_rows": 141, "positive_rows": 151, "precision": 0.8299319727891157, "recall": 0.8079470198675497, "rows": 292 } }, "overall": { "accuracy": 0.8244886677722498, "confusion_matrix": { "fn": 394, "fp": 241, "tn": 1590, "tp": 1393 }, "f1": 0.8143817597193804, "false_positive_rate": 0.1316220644456581, "false_positive_rate_wilson_95": { "lower": 0.11690490414017769, "upper": 0.14788171079327733 }, "negative_rows": 1831, "positive_rows": 1787, "precision": 0.8525091799265606, "recall": 0.7795187465025182, "rows": 3618 }, "source_query_id": "56061a8b-5c0b-458b-80f3-31977b9352da" }, "14_temporal_random_forest": { "by_week_id": { "52": { "accuracy": 0.8939929328621908, "confusion_matrix": { "fn": 23, "fp": 7, "tn": 147, "tp": 106 }, "f1": 0.8760330578512397, "false_positive_rate": 0.045454545454545456, "false_positive_rate_wilson_95": { "lower": 0.02219003374649918, "upper": 0.09084401291016404 }, "negative_rows": 154, "positive_rows": 129, "precision": 0.9380530973451328, "recall": 0.8217054263565892, "rows": 283 }, "53": { "accuracy": 0.9174041297935103, "confusion_matrix": { "fn": 19, "fp": 9, "tn": 169, "tp": 142 }, "f1": 0.9102564102564102, "false_positive_rate": 0.05056179775280899, "false_positive_rate_wilson_95": { "lower": 0.026825714697326877, "upper": 0.09328693157346127 }, "negative_rows": 178, "positive_rows": 161, "precision": 0.9403973509933775, "recall": 0.8819875776397516, "rows": 339 }, "54": { "accuracy": 0.8833333333333333, "confusion_matrix": { "fn": 28, "fp": 7, "tn": 145, "tp": 120 }, "f1": 0.8727272727272728, "false_positive_rate": 0.046052631578947366, "false_positive_rate_wilson_95": { "lower": 0.0224843353940877, "upper": 0.09200033996577209 }, "negative_rows": 152, "positive_rows": 148, "precision": 0.9448818897637795, "recall": 0.8108108108108109, "rows": 300 }, "55": { "accuracy": 0.9259259259259259, "confusion_matrix": { "fn": 14, "fp": 8, "tn": 132, "tp": 143 }, "f1": 0.9285714285714285, "false_positive_rate": 0.05714285714285714, "false_positive_rate_wilson_95": { "lower": 0.029236485290990835, "upper": 0.10870329217906068 }, "negative_rows": 140, "positive_rows": 157, "precision": 0.9470198675496688, "recall": 0.910828025477707, "rows": 297 }, "56": { "accuracy": 0.889589905362776, "confusion_matrix": { "fn": 25, "fp": 10, "tn": 138, "tp": 144 }, "f1": 0.8916408668730651, "false_positive_rate": 0.06756756756756757, "false_positive_rate_wilson_95": { "lower": 0.03711222112207073, "upper": 0.1199032541365536 }, "negative_rows": 148, "positive_rows": 169, "precision": 0.935064935064935, "recall": 0.8520710059171598, "rows": 317 }, "57": { "accuracy": 0.9014084507042254, "confusion_matrix": { "fn": 16, "fp": 12, "tn": 136, "tp": 120 }, "f1": 0.8955223880597014, "false_positive_rate": 0.08108108108108109, "false_positive_rate_wilson_95": { "lower": 0.04698837084876442, "upper": 0.1363703708080279 }, "negative_rows": 148, "positive_rows": 136, "precision": 0.9090909090909091, "recall": 0.8823529411764706, "rows": 284 }, "58": { "accuracy": 0.9310344827586207, "confusion_matrix": { "fn": 14, "fp": 6, "tn": 144, "tp": 126 }, "f1": 0.9264705882352942, "false_positive_rate": 0.04, "false_positive_rate_wilson_95": { "lower": 0.01845916060207442, "upper": 0.08451346421655606 }, "negative_rows": 150, "positive_rows": 140, "precision": 0.9545454545454546, "recall": 0.9, "rows": 290 }, "59": { "accuracy": 0.884375, "confusion_matrix": { "fn": 26, "fp": 11, "tn": 158, "tp": 125 }, "f1": 0.8710801393728224, "false_positive_rate": 0.0650887573964497, "false_positive_rate_wilson_95": { "lower": 0.03672990367085314, "upper": 0.11277970471252713 }, "negative_rows": 169, "positive_rows": 151, "precision": 0.9191176470588235, "recall": 0.8278145695364238, "rows": 320 }, "60": { "accuracy": 0.8612099644128114, "confusion_matrix": { "fn": 30, "fp": 9, "tn": 139, "tp": 103 }, "f1": 0.8408163265306122, "false_positive_rate": 0.060810810810810814, "false_positive_rate_wilson_95": { "lower": 0.03231935672911554, "upper": 0.1115244853304248 }, "negative_rows": 148, "positive_rows": 133, "precision": 0.9196428571428571, "recall": 0.7744360902255639, "rows": 281 }, "61": { "accuracy": 0.845679012345679, "confusion_matrix": { "fn": 38, "fp": 12, "tn": 142, "tp": 132 }, "f1": 0.8407643312101911, "false_positive_rate": 0.07792207792207792, "false_positive_rate_wilson_95": { "lower": 0.04513428609205353, "upper": 0.1312544715177052 }, "negative_rows": 154, "positive_rows": 170, "precision": 0.9166666666666666, "recall": 0.7764705882352941, "rows": 324 }, "62": { "accuracy": 0.8865979381443299, "confusion_matrix": { "fn": 21, "fp": 12, "tn": 137, "tp": 121 }, "f1": 0.88, "false_positive_rate": 0.08053691275167785, "false_positive_rate_wilson_95": { "lower": 0.046668847532353515, "upper": 0.13549022786194978 }, "negative_rows": 149, "positive_rows": 142, "precision": 0.9097744360902256, "recall": 0.852112676056338, "rows": 291 }, "63": { "accuracy": 0.9143835616438356, "confusion_matrix": { "fn": 16, "fp": 9, "tn": 132, "tp": 135 }, "f1": 0.9152542372881357, "false_positive_rate": 0.06382978723404255, "false_positive_rate_wilson_95": { "lower": 0.033941271284132206, "upper": 0.11685435725018589 }, "negative_rows": 141, "positive_rows": 151, "precision": 0.9375, "recall": 0.8940397350993378, "rows": 292 } }, "overall": { "accuracy": 0.8944168048645661, "confusion_matrix": { "fn": 270, "fp": 112, "tn": 1719, "tp": 1517 }, "f1": 0.8881733021077284, "false_positive_rate": 0.06116876024030585, "false_positive_rate_wilson_95": { "lower": 0.051084113107108184, "upper": 0.07309089815906107 }, "negative_rows": 1831, "positive_rows": 1787, "precision": 0.9312461632903621, "recall": 0.8489087856743145, "rows": 3618 }, "source_query_id": "80ab6789-7385-4af0-9a5d-f6c2f177143b" } }, "label_semantics": { "negative": 0, "positive": 1 }, "official_full_feature_reference": { "Win32": { "accuracy": 0.9806522940851299, "confusion_matrix": { "fn": 53, "fp": 17, "tn": 1814, "tp": 1734 }, "f1": 0.9802148106274731, "false_positive_rate": 0.009284543965046422, "false_positive_rate_wilson_95": { "lower": 0.005804885864935755, "upper": 0.014818944405945483 }, "negative_rows": 1831, "positive_rows": 1787, "precision": 0.9902912621359223, "recall": 0.970341354224958, "rows": 3618 }, "comparison_note": "Reference only. Training data and feature dimensions differ from the query-local histogram classifiers.", "feature_count": 2568, "source_query_id": "3c3ae508-178f-4f15-9ab5-c483ba987231", "threshold": 0.5, "training_corpus": "EMBER2024_all.model trained with a larger, full upstream corpus" } } }