Bertindaklah sebagai Senior Data Scientist & Pakar...

python
df = pd.read_csv('/mnt/data/PRIZE_LEVEL_4_ELITE.csv')
df.head()

             timestamp  raw_number  d1  ...  rolling_mean_sum_3     h_sin     h_cos
0  2026-02-10 21:30:00        6112   6  ...            0.000000 -0.608761  0.793353
1  2026-02-10 21:30:00         236   0  ...            0.000000 -0.608761  0.793353
2  2026-02-10 21:30:00        7341   7  ...           12.000000 -0.608761  0.793353
3  2026-02-11 03:30:00        9876   9  ...           18.666667  0.793353  0.608761
4  2026-02-11 03:30:00        2992   2  ...           22.333333  0.793353  0.608761

[5 rows x 21 columns]

python
list(df.columns)

['timestamp',
 'raw_number',
 'd1',
 'd2',
 'd3',
 'd4',
 'sum_digits',
 'p1',
 'p2',
 'p3',
 'p4',
 'm1',
 'm2',
 'm3',
 'm4',
 'is_double',
 'lag_1_sum',
 'lag_2_sum',
 'rolling_mean_sum_3',
 'h_sin',
 'h_cos']

python
df['timestamp'] = pd.to_datetime(df['timestamp'])
df['timeslot'] = df['timestamp'].dt.strftime('%H:%M')
df['timeslot'].value_counts().head()

timeslot
21:30    360
03:30    360
06:30    360
11:30    357
17:30    357
Name: count, dtype: int64

python
df = df.sort_values('timestamp').reset_index(drop=True)
df.tail()

               timestamp  raw_number  d1  ...     h_sin     h_cos  timeslot
2128 2026-06-10 03:30:00        7966   7  ...  0.793353  0.608761     03:30
2129 2026-06-10 03:30:00        8741   8  ...  0.793353  0.608761     03:30
2130 2026-06-10 06:30:00        7301   7  ...  0.991445 -0.130526     06:30
2131 2026-06-10 06:30:00        3261   3  ...  0.991445 -0.130526     06:30
2132 2026-06-10 06:30:00         913   0  ...  0.991445 -0.130526     06:30

[5 rows x 22 columns]

python
test_weights = {'freq':1,'gap':1,'corr':1,'sum':1,'odd_even':1,'pattern':1}
print(evaluate_weights(test_weights, 150))

python
import random, math
def random_weight_search(num_iter=100, recent_n=150):
    best_metric = float('inf')
    best_weights = None
    best_acc = 0
    best_median = None
    for _ in range(num_iter):
        weights = {k: random.random() for k in ['freq','gap','corr','sum','odd_even','pattern']}
        top10_acc, median_rank = evaluate_weights(weights, recent_n)
        metric = (1 - top10_acc) + median_rank/10000.0
        if metric < best_metric:
            best_metric = metric
            best_weights = weights
            best_acc = top10_acc
            best_median = median_rank
    return best_weights, best_acc, best_median
best_weights, best_acc, best_median = random_weight_search(30, 150)
best_weights, best_acc, best_median

python
import time, random
start = time.time()
evaluate_weights(test_weights, 50)
print("Time:", time.time()-start)

python
start = time.time()
evaluate_weights(test_weights, 150)
print("Time:", time.time()-start)

python
def random_weight_search(num_iter=10, recent_n=150):
    best_metric = float('inf')
    best_weights = None
    best_acc = 0
    best_median = None
    for _ in range(num_iter):
        weights = {k: random.random() for k in ['freq','gap','corr','sum','odd_even','pattern']}
        top10_acc, median_rank = evaluate_weights(weights, recent_n)
        metric = (1 - top10_acc) + median_rank/10000.0
        if metric < best_metric:
            best_metric = metric
            best_weights = weights
            best_acc = top10_acc
            best_median = median_rank
    return best_weights, best_acc, best_median, best_metric

best_w, acc, med, met = random_weight_search(10,150)
best_w, acc, med

python
start = time.time()
evaluate_weights_fast(test_weights, 150)
print("time:", time.time()-start)

python
evaluate_weights_fast(test_weights, 150)

(0.0033333333333333335, 3739.5)

python
def random_weight_search_fast(num_iter=50, recent_n=150):
    best_metric = float('inf')
    best_weights = None
    best_acc = 0
    best_median = None
    for _ in range(num_iter):
        weights = {k: random.random() for k in ['freq','gap','corr','sum','odd_even','pattern']}
        top10_acc, median_rank = evaluate_weights_fast(weights, recent_n)
        metric = (1 - top10_acc) + median_rank/10000.0
        if metric < best_metric:
            best_metric = metric
            best_weights = weights
            best_acc = top10_acc
            best_median = median_rank
    return best_weights, best_acc, best_median
start = time.time()
bw, ba, bm = random_weight_search_fast(100,150)
print("time", time.time()-start)
bw, ba, bm

python
pattern_id_array_candidates = np.array([pattern_map[pattern_category(tuple(digits_array[i]))] for i in range(10000)], dtype=np.int8)
print(np.unique(pattern_id_array_candidates, return_counts=True))

python
start = time.time()
evaluate_weights_fast(test_weights, 150)
print("time:", time.time()-start)

python
def random_weight_search_fast(num_iter=200, recent_n=150):
    best_metric = float('inf')
    best_weights = None
    best_acc = 0
    best_median = None
    for _ in range(num_iter):
        weights = {k: random.random() for k in ['freq','gap','corr','sum','odd_even','pattern']}
        top10_acc, median_rank = evaluate_weights_fast(weights, recent_n)
        metric = (1 - top10_acc) + median_rank/10000.0
        if metric < best_metric:
            best_metric = metric
            best_weights = weights
            best_acc = top10_acc
            best_median = median_rank
    return best_weights, best_acc, best_median, best_metric

start = time.time()
bw, ba, bm, bmmet = random_weight_search_fast(200,150)
print("time:", time.time()-start)
bw, ba, bm

python
start = time.time()
bw, ba, bm, _ = random_weight_search_fast(100,150)
print("time:", time.time()-start)
bw, ba, bm

({'freq': 0.11584457497815648,
  'gap': 0.006229964337335248,
  'corr': 0.42007869332616965,
  'sum': 0.7247475908969793,
  'odd_even': 0.09044593154770098,
  'pattern': 0.7598256260138564},
 0.0033333333333333335,
 3618.0)

python
def tune_recent(recent_vals=[50,150,300], iterations=100):
    best_overall = None
    best_metric = float('inf')
    results = {}
    for recent_n in recent_vals:
        start = time.time()
        w, acc, med, metric = random_weight_search_fast(iterations, recent_n)
        duration = time.time()-start
        results[recent_n]= (w, acc, med, metric, duration)
        print("recent",recent_n,"duration",duration,"acc",acc,"med",med)
        if metric < best_metric:
            best_metric = metric
            best_overall = (recent_n, w, acc, med)
    return best_overall, results

best_overall, results = tune_recent([50,150,300], 50)

python
best_recent, best_weights, best_acc, best_med = best_overall
print(best_recent, best_weights)
# compute predictions using full latest window best_recent
start_idx = len(df)-best_recent
window_numbers = numbers_array[start_idx:]
# compute freq etc
freq_counts = np.bincount(window_numbers, minlength=10000).astype(np.float32)
freq_score = freq_counts / best_recent
gap_idx = np.full(10000, -1, dtype=np.int32)
for pos_in_window, num in enumerate(window_numbers):
    gap_idx[num] = pos_in_window
gap_score = np.where(gap_idx>=0, 1.0/(len(window_numbers)-gap_idx), 0.0)
sum_counts = np.bincount(sum_array[start_idx:], minlength=37).astype(np.float32)
sum_score = sum_counts[sum_digits_array] / best_recent
odd_counts = np.bincount(odd_count_array[start_idx:], minlength=5).astype(np.float32)
odd_score = odd_counts[odd_even_array] / best_recent
pattern_counts = np.bincount(pattern_index_array[start_idx:], minlength=5).astype(np.float32)
pattern_score = pattern_counts[pattern_id_array_candidates] / best_recent
# corr score for latest draw to slot not known; we can use last draw to determine prev digits for next predictions if next draw same slot as last draw.
latest_row = df.iloc[-1]
current_slot = latest_row['timeslot']
prev_digits_for_next = (latest_row['d1'], latest_row['d2'], latest_row['d3'], latest_row['d4'])
matrices = markov_probs_dict[current_slot]
prob = matrices[0][prev_digits_for_next[0], digits_array[:,0]]
for pos in range(1,4):
    prob *= matrices[pos][prev_digits_for_next[pos], digits_array[:,pos]]
corr_score = prob
total_score = (
    best_weights['freq']*freq_score +
    best_weights['gap']*gap_score +
    best_weights['corr']*corr_score +
    best_weights['sum']*sum_score +
    best_weights['odd_even']*odd_score +
    best_weights['pattern']*pattern_score
)
# top 10
top10_idx = np.argsort(-total_score)[:10]
top10_numbers = ["{:04d}".format(i) for i in top10_idx]
top10_scores = total_score[top10_idx]
top10_numbers, top10_scores

(['0913',
  '8741',
  '3261',
  '8570',
  '7301',
  '0359',
  '6815',
  '7094',
  '2387',
  '8075'],
 array([0.63185774, 0.44960578, 0.44704336, 0.41541424, 0.40596084,
        0.39393489, 0.3860056 , 0.38320507, 0.37860105, 0.37832308]))

python
baseline_weights = {'freq':1,'gap':1,'corr':1,'sum':1,'odd_even':1,'pattern':1}
baseline_acc, baseline_med = evaluate_weights_fast(baseline_weights, best_recent)
baseline_acc, baseline_med

(0.0033333333333333335, 3842.5)

python
import pandas as pd
import numpy as np
import random
import matplotlib.pyplot as plt
from collections import defaultdict, Counter

# === Load data ===
file_path = "/mnt/data/PRIZE_LEVEL_4_ELITE.csv"
df = pd.read_csv(file_path, parse_dates=["timestamp"])

# === Helper precomputations ===
df["timeslot"] = df["timestamp"].dt.strftime("%H:%M")
numbers_array = df["raw_number"].values
sum_array = df["sum_digits"].values
odd_count_array = (df[["d1", "d2", "d3", "d4"]].values % 2).sum(axis=1)

# Digit decomposition for all 4‑digit numbers
def number_to_digits(n: int):
    return (n // 1000, (n // 100) % 10, (n // 10) % 10, n % 10)

digits_array = np.array([number_to_digits(i) for i in range(10_000)], dtype=np.int8)
sum_digits_array = digits_array.sum(axis=1)
odd_even_array = (digits_array % 2).sum(axis=1)

# Pattern categories
def pattern_category(d):
    a, b, c, e = d
    cnt = Counter(d)
    if len(cnt) == 1:
        return "AAAA"
    if 3 in cnt.values():
        return "AAAB"
    if sorted(cnt.values()) == [2, 2]:
        return "AABB"
    if 2 in cnt.values():
        return "AABC"
    return "ABCD"

pattern_map = {"AAAA": 0, "AAAB": 1, "AABB": 2, "AABC": 3, "ABCD": 4}
pattern_id_array_candidates = np.array(
    [pattern_map[pattern_category(tuple(d))] for d in digits_array], dtype=np.int8
)
pattern_index_array = np.array(
    [pattern_map[pattern_category((r.d1, r.d2, r.d3, r.d4))] for r in df.itertuples()],
    dtype=np.int8,
)

# Pre‑compute previous digits per slot and global Markov probabilities
prev_digits_array = np.zeros((len(df), 4), dtype=np.int8)
markov_dict = defaultdict(lambda: [np.ones((10, 10), dtype=np.float32) for _ in range(4)])
last_digits_per_slot = {}
for i, r in enumerate(df.itertuples()):
    slot = r.timeslot
    cur_digits = (r.d1, r.d2, r.d3, r.d4)
    if slot in last_digits_per_slot:
        prev_digits_array[i] = last_digits_per_slot[slot]
    last_digits_per_slot[slot] = cur_digits

    # Build Markov counts only on data prior to back‑test window (exclude last 300 rows)
    if i < len(df) - 300 and prev_digits_array[i].sum() != 0:
        for pos in range(4):
            markov_dict[slot][pos][prev_digits_array[i][pos], cur_digits[pos]] += 1

markov_probs_dict = {
    slot: [m / m.sum(axis=1, keepdims=True) for m in mats] for slot, mats in markov_dict.items()
}

# === Scoring & evaluation ===
def evaluate_weights_fast(w, recent_n):
    eval_start = len(df) - 300
    hits, ranks = 0, []
    for idx in range(eval_start, len(df)):
        start_idx = max(0, idx - recent_n)
        window_numbers = numbers_array[start_idx:idx]

        freq_counts = np.bincount(window_numbers, minlength=10_000).astype(np.float32)
        freq = freq_counts / recent_n

        gap_idx = np.full(10_000, -1, dtype=np.int32)
        for pos, num in enumerate(window_numbers):
            gap_idx[num] = pos
        gap = np.where(gap_idx >= 0, 1.0 / (len(window_numbers) - gap_idx), 0.0)

        sum_counts = np.bincount(sum_array[start_idx:idx], minlength=37).astype(np.float32)
        sum_feat = sum_counts[sum_digits_array] / recent_n

        odd_counts = np.bincount(odd_count_array[start_idx:idx], minlength=5).astype(np.float32)
        odd_feat = odd_counts[odd_even_array] / recent_n

        pattern_counts = np.bincount(pattern_index_array[start_idx:idx], minlength=5).astype(
            np.float32
        )
        pattern_feat = pattern_counts[pattern_id_array_candidates] / recent_n

        slot = df.at[idx, "timeslot"]
        prev_d = prev_digits_array[idx]
        mats = markov_probs_dict[slot]
        corr = mats[0][prev_d[0], digits_array[:, 0]].copy()
        for p in range(1, 4):
            corr *= mats[p][prev_d[p], digits_array[:, p]]

        score = (
            w["freq"] * freq
            + w["gap"] * gap
            + w["corr"] * corr
            + w["sum"] * sum_feat
            + w["odd_even"] * odd_feat
            + w["pattern"] * pattern_feat
        )

        actual = numbers_array[idx]
        rank = (score > score[actual]).sum() + 1
        ranks.append(rank)
        if rank <= 10:
            hits += 1

    return hits / 300.0, float(np.median(ranks))


def random_weight_search(iterations, recent_n):
    best_w = None
    best_score = np.inf
    best_acc, best_med = 0, None
    for _ in range(iterations):
        w = {k: random.random() for k in ["freq", "gap", "corr", "sum", "odd_even", "pattern"]}
        acc, med = evaluate_weights_fast(w, recent_n)
        metric = (1 - acc) + med / 10_000.0  # combine two objectives
        if metric < best_score:
            best_score = metric
            best_w, best_acc, best_med = w, acc, med
    return best_w, best_acc, best_med, best_score


# === Baseline (equal weights) ===
baseline_w = {k: 1.0 for k in ["freq", "gap", "corr", "sum", "odd_even", "pattern"]}
baseline_acc, baseline_med = evaluate_weights_fast(baseline_w, 150)

# === Hyper‑parameter tuning ===
search_results = {}
for recent in (50, 150, 300):
    best_w, best_acc, best_med, _ = random_weight_search(50, recent)
    search_results[recent] = {"weights": best_w, "acc": best_acc, "median": best_med}

# Pick overall best by the same metric used in tuning
overall_best_recent = min(
    search_results.keys(),
    key=lambda r: (1 - search_results[r]["acc"]) + search_results[r]["median"] / 10_000.0,
)
best_w = search_results[overall_best_recent]["weights"]

# === Final prediction for next draw ===
start_idx = len(df) - overall_best_recent
window_numbers = numbers_array[start_idx:]

freq_counts = np.bincount(window_numbers, minlength=10_000).astype(np.float32)
freq = freq_counts / overall_best_recent

gap_idx = np.full(10_000, -1, dtype=np.int32)
for pos, num in enumerate(window_numbers):
    gap_idx[num] = pos
gap = np.where(gap_idx >= 0, 1.0 / (len(window_numbers) - gap_idx), 0.0)

sum_counts = np.bincount(sum_array[start_idx:], minlength=37).astype(np.float32)
sum_feat = sum_counts[sum_digits_array] / overall_best_recent

odd_counts = np.bincount(odd_count_array[start_idx:], minlength=5).astype(np.float32)
odd_feat = odd_counts[odd_even_array] / overall_best_recent

pattern_counts = np.bincount(pattern_index_array[start_idx:], minlength=5).astype(np.float32)
pattern_feat = pattern_counts[pattern_id_array_candidates] / overall_best_recent

last_row = df.iloc[-1]
slot = last_row["timeslot"]
prev_d_for_next = (last_row.d1, last_row.d2, last_row.d3, last_row.d4)
mats = markov_probs_dict[slot]
corr = mats[0][prev_d_for_next[0], digits_array[:, 0]].copy()
for p in range(1, 4):
    corr *= mats[p][prev_d_for_next[p], digits_array[:, p]]

final_score = (
    best_w["freq"] * freq
    + best_w["gap"] * gap
    + best_w["corr"] * corr
    + best_w["sum"] * sum_feat
    + best_w["odd_even"] * odd_feat
    + best_w["pattern"] * pattern_feat
)

top10_idx = np.argsort(-final_score)[:10]
top10_numbers = [f"{i:04d}" for i in top10_idx]
top10_scores = final_score[top10_idx]

# === Results summary table ===
summary_df = pd.DataFrame(
    {
        "Scenario": ["Baseline (equal w)", "Optimized"],
        "Top‑10 Accuracy": [baseline_acc, search_results[overall_best_recent]["acc"]],
        "Median Rank": [baseline_med, search_results[overall_best_recent]["median"]],
    }
)

# Display summary as an interactive table
import caas_jupyter_tools as cj

cj.display_dataframe_to_user("Performance Summary", summary_df)

# Simple bar chart for median rank comparison
plt.figure(figsize=(6, 4))
plt.bar(summary_df["Scenario"], summary_df["Median Rank"])
plt.title("Median Rank Before vs After Optimisation")
plt.ylabel("Median Rank (lower is better)")
plt.tight_layout()
plt.show()

# Print key outputs
print("===== Best Hyper‑Parameters =====")
print(f"Recent window (n): {overall_best_recent}")
print("Weights:", {k: round(v, 4) for k, v in best_w.items()})
print("\n===== Top‑10 Candidates For Next Draw =====")
for num, sc in zip(top10_numbers, top10_scores):
    print(f"{num}  |  score={sc:.4f}")