Upload 4 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+no-HH-LL/market_representation_M3_high_res.png filter=lfs diff=lfs merge=lfs -text
+with-HH-LL/market_representation_M3_high_res.png filter=lfs diff=lfs merge=lfs -text

no-HH-LL/no_highest-high_no_lowest-low.py

@@ -0,0 +1,203 @@
+# !pip install pandas numpy matplotlib
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib.collections import LineCollection, PolyCollection
+import time
+
+class MarketRepresentation:
+    def __init__(self, gap, premarket, daytime, nighttime):
+        self.current_time_gap = gap
+        self.current_premarket_session = premarket
+        self.current_daytime_session = daytime
+        self.current_nighttime_session = nighttime
+        
+    def to_dataframe(self):
+        """Reconstruct the active time session."""
+        return pd.concat([
+            self.current_premarket_session, 
+            self.current_daytime_session, 
+            self.current_nighttime_session
+        ])
+
+def run_market_representation_pipeline(csv_path):
+    print("="*55)
+    print("1. LOADING DATA")
+    print("="*55)
+    start_time = time.time()
+    
+    df = pd.read_csv(csv_path)
+    # Optimized datetime parsing
+    df['datetime'] = pd.to_datetime(df['datetime'], format='%Y-%m-%d %H:%M:%S')
+    df = df.sort_values('datetime').reset_index(drop=True)
+    
+    print(f"[✔] Data loaded in {time.time() - start_time:.2f} seconds.")
+    print(f"[i] Total records: {len(df):,}")
+    
+    print("\n" + "="*55)
+    print("2. EXTRACTING COMPLETE DAILY REPRESENTATIONS")
+    print("="*55)
+    extract_start = time.time()
+    
+    # 45 mins threshold robustly captures the standard 1-hour daily break and 48-hour weekend gap
+    gap_threshold = pd.Timedelta(minutes=45)
+    
+    df['time_diff'] = df['datetime'].diff()
+    is_gap = df['time_diff'] > gap_threshold
+    df['session_id'] = is_gap.cumsum()
+    
+    representations = []
+    
+    grouped = df.groupby('session_id')
+    for sid, group in grouped:
+        # 1. current_time_gap
+        current_time_gap = group['time_diff'].iloc[0] if sid > 0 else pd.Timedelta(seconds=0)
+        
+        # Determine reference date for 0:00 UTC
+        zero_hour_rows = group[group['datetime'].dt.hour == 0]
+        if len(zero_hour_rows) > 0:
+            ref_date = zero_hour_rows['datetime'].dt.floor('D').iloc[0]
+        else:
+            # Fallback for sessions that skip exactly 0:00 but span across it
+            ref_date = group['datetime'].iloc[len(group)//2].floor('D')
+            
+        group = group.copy()
+        
+        # Relative time in hours (0:00 UTC is exactly 0.0)
+        group['rel_time_hours'] = (group['datetime'] - ref_date).dt.total_seconds() / 3600.0
+        
+        # Align all candle_open at 0:00 UTC to Y=0
+        idx_closest_to_zero = group['rel_time_hours'].abs().idxmin()
+        ref_price = group.loc[idx_closest_to_zero, 'open']
+        
+        group['open'] -= ref_price
+        group['high'] -= ref_price
+        group['low'] -= ref_price
+        group['close'] -= ref_price
+        
+        # Slicing sessions based on precise UTC constraints
+        # 2. current_premarket_session
+        current_premarket_session = group[group['rel_time_hours'] < 0.0]
+        # 3. current_daytime_session
+        current_daytime_session = group[(group['rel_time_hours'] >= 0.0) & (group['rel_time_hours'] < 12.0)]
+        # 4. current_nighttime_session
+        current_nighttime_session = group[group['rel_time_hours'] >= 12.0]
+        
+        # Keep only structurally complete representations
+        if not current_premarket_session.empty and not current_daytime_session.empty and not current_nighttime_session.empty:
+            rep = MarketRepresentation(
+                current_time_gap,
+                current_premarket_session,
+                current_daytime_session,
+                current_nighttime_session
+            )
+            representations.append(rep)
+            
+    print(f"[✔] Extraction complete in {time.time() - extract_start:.2f} seconds.")
+    print(f"[i] Found {len(representations):,} complete daily market representations.")
+    
+    if len(representations) == 0:
+        print("[!] No complete sessions found. Exiting.")
+        return
+
+    print("\n" + "="*55)
+    print("3. GENERATING VECTORIZED OVERLAY PLOT")
+    print("="*55)
+    plot_start = time.time()
+    
+    # Fast merge of validated data structures
+    valid_sessions_dfs = [rep.to_dataframe() for rep in representations]
+    df_aligned = pd.concat(valid_sessions_dfs, ignore_index=True)
+    
+    up = df_aligned['close'] >= df_aligned['open']
+    down = ~up
+    
+    # Render width parameters UPDATED FOR M3 (3 minutes)
+    width = (3.0 / 60.0) * 0.8  # 80% width of a 3-minute period mapped in hours
+    w = width / 2
+    
+    x = df_aligned['rel_time_hours'].values
+    o = df_aligned['open'].values
+    c = df_aligned['close'].values
+    h = df_aligned['high'].values
+    l = df_aligned['low'].values
+    
+    x_up, o_up, c_up, h_up, l_up = x[up], o[up], c[up], h[up], l[up]
+    x_down, o_down, c_down, h_down, l_down = x[down], o[down], c[down], h[down], l[down]
+    
+    # Memory-efficient vectorized polygon coordinate mapping
+    wicks_up = np.empty((len(x_up), 2, 2))
+    wicks_up[:, 0, 0] = x_up; wicks_up[:, 0, 1] = l_up
+    wicks_up[:, 1, 0] = x_up; wicks_up[:, 1, 1] = h_up
+    
+    wicks_down = np.empty((len(x_down), 2, 2))
+    wicks_down[:, 0, 0] = x_down; wicks_down[:, 0, 1] = l_down
+    wicks_down[:, 1, 0] = x_down; wicks_down[:, 1, 1] = h_down
+    
+    boxes_up = np.empty((len(x_up), 4, 2))
+    boxes_up[:, 0, 0] = x_up - w; boxes_up[:, 0, 1] = o_up
+    boxes_up[:, 1, 0] = x_up + w; boxes_up[:, 1, 1] = o_up
+    boxes_up[:, 2, 0] = x_up + w; boxes_up[:, 2, 1] = c_up
+    boxes_up[:, 3, 0] = x_up - w; boxes_up[:, 3, 1] = c_up
+    
+    boxes_down = np.empty((len(x_down), 4, 2))
+    boxes_down[:, 0, 0] = x_down - w; boxes_down[:, 0, 1] = o_down
+    boxes_down[:, 1, 0] = x_down + w; boxes_down[:, 1, 1] = o_down
+    boxes_down[:, 2, 0] = x_down + w; boxes_down[:, 2, 1] = c_down
+    boxes_down[:, 3, 0] = x_down - w; boxes_down[:, 3, 1] = c_down
+    
+    # White-themed rendering (Massive Canvas for 1200 DPI Detail)
+    plt.style.use('default')
+    fig, ax = plt.subplots(figsize=(30, 15), facecolor='white')
+    ax.set_facecolor('white')
+    
+    up_color = '#26a69a'
+    down_color = '#ef5350'
+    alpha_val = 0.1  # Exactly 10% opacity
+    
+    # Hardware-accelerated GPU/CPU rendering using Collections
+    lc_up = LineCollection(wicks_up, colors=up_color, linewidths=0.5, alpha=alpha_val)
+    lc_down = LineCollection(wicks_down, colors=down_color, linewidths=0.5, alpha=alpha_val)
+    pc_up = PolyCollection(boxes_up, facecolors=up_color, edgecolors='none', alpha=alpha_val)
+    pc_down = PolyCollection(boxes_down, facecolors=down_color, edgecolors='none', alpha=alpha_val)
+    
+    ax.add_collection(lc_up)
+    ax.add_collection(lc_down)
+    ax.add_collection(pc_up)
+    ax.add_collection(pc_down)
+    
+    ax.autoscale_view()
+    
+    # Annotations & Formatting
+    ax.set_title("Complete Daily Market Representation Overlay (M3 Timeframe)\nAligned at 0:00 UTC", fontsize=20, fontweight='bold', pad=15)
+    ax.set_xlabel("Hours Relative to 0:00 UTC", fontsize=16, labelpad=10)
+    ax.set_ylabel("Price Deviation from 0:00 UTC Open", fontsize=16, labelpad=10)
+    
+    min_x, max_x = ax.get_xlim()
+    ax.set_xticks(np.arange(np.floor(min_x), np.ceil(max_x)+1, 2))
+    
+    # Segment shading
+    ax.axvspan(min_x, 0, color='#e3f2fd', alpha=0.3, label='Premarket (< 0:00)')
+    ax.axvspan(0, 12, color='#fff3e0', alpha=0.3, label='Daytime (0:00 - 12:00)')
+    ax.axvspan(12, max_x, color='#f3e5f5', alpha=0.3, label='Nighttime (> 12:00)')
+    
+    ax.grid(True, linestyle='--', alpha=0.5)
+    ax.axvline(0, color='black', linestyle='-', linewidth=2.0, alpha=0.8, label='0:00 UTC Alignment')
+    ax.axhline(0, color='black', linestyle='-', linewidth=2.0, alpha=0.8)
+    
+    ax.legend(loc='upper left', fontsize=14)
+    plt.tight_layout()
+    
+    # High Resolution Export Setup
+    export_path = 'market_representation_M3_high_res.png'
+    print(f"[✔] Exporting Extreme-Res image (1200 DPI) to {export_path}...")
+    plt.savefig(export_path, dpi=1200, bbox_inches='tight')
+    print(f"[✔] Plot generated and saved in {time.time() - plot_start:.2f} seconds.")
+    print("="*55)
+    plt.show()
+
+# Execution Call
+# Upload your M3 file to Colab and update the path accordingly.
+csv_path = '/content/XAUUSDc_M3_2014-01-14_to_2026-04-29_1060828records_88092KB.csv'
+run_market_representation_pipeline(csv_path)

with-HH-LL/wtih_highest-high_no_lowest-low.py

@@ -0,0 +1,241 @@
+# !pip install pandas numpy matplotlib
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib.collections import LineCollection, PolyCollection
+import time
+
+class MarketRepresentation:
+    def __init__(self, gap, premarket, daytime, nighttime, extremes=None):
+        self.current_time_gap = gap
+        self.current_premarket_session = premarket
+        self.current_daytime_session = daytime
+        self.current_nighttime_session = nighttime
+        self.extremes = extremes
+        
+    def to_dataframe(self):
+        """Reconstruct the active time session."""
+        return pd.concat([
+            self.current_premarket_session, 
+            self.current_daytime_session, 
+            self.current_nighttime_session
+        ])
+
+def run_market_representation_pipeline(csv_path):
+    print("="*55)
+    print("1. LOADING DATA")
+    print("="*55)
+    start_time = time.time()
+    
+    df = pd.read_csv(csv_path)
+    # Optimized datetime parsing
+    df['datetime'] = pd.to_datetime(df['datetime'], format='%Y-%m-%d %H:%M:%S')
+    df = df.sort_values('datetime').reset_index(drop=True)
+    
+    print(f"[✔] Data loaded in {time.time() - start_time:.2f} seconds.")
+    print(f"[i] Total records: {len(df):,}")
+    
+    print("\n" + "="*55)
+    print("2. EXTRACTING COMPLETE DAILY REPRESENTATIONS")
+    print("="*55)
+    extract_start = time.time()
+    
+    # 45 mins threshold robustly captures the standard 1-hour daily break and 48-hour weekend gap
+    gap_threshold = pd.Timedelta(minutes=45)
+    
+    df['time_diff'] = df['datetime'].diff()
+    is_gap = df['time_diff'] > gap_threshold
+    df['session_id'] = is_gap.cumsum()
+    
+    representations = []
+    
+    grouped = df.groupby('session_id')
+    for sid, group in grouped:
+        # 1. current_time_gap
+        current_time_gap = group['time_diff'].iloc[0] if sid > 0 else pd.Timedelta(seconds=0)
+        
+        # Determine reference date for 0:00 UTC
+        zero_hour_rows = group[group['datetime'].dt.hour == 0]
+        if len(zero_hour_rows) > 0:
+            ref_date = zero_hour_rows['datetime'].dt.floor('D').iloc[0]
+        else:
+            # Fallback for sessions that skip exactly 0:00 but span across it
+            ref_date = group['datetime'].iloc[len(group)//2].floor('D')
+            
+        group = group.copy()
+        
+        # Relative time in hours (0:00 UTC is exactly 0.0)
+        group['rel_time_hours'] = (group['datetime'] - ref_date).dt.total_seconds() / 3600.0
+        
+        # Align all candle_open at 0:00 UTC to Y=0
+        idx_closest_to_zero = group['rel_time_hours'].abs().idxmin()
+        ref_price = group.loc[idx_closest_to_zero, 'open']
+        
+        group['open'] -= ref_price
+        group['high'] -= ref_price
+        group['low'] -= ref_price
+        group['close'] -= ref_price
+        
+        # Slicing sessions based on precise UTC constraints
+        # 2. current_premarket_session
+        current_premarket_session = group[group['rel_time_hours'] < 0.0]
+        # 3. current_daytime_session
+        current_daytime_session = group[(group['rel_time_hours'] >= 0.0) & (group['rel_time_hours'] < 12.0)]
+        # 4. current_nighttime_session
+        current_nighttime_session = group[group['rel_time_hours'] >= 12.0]
+        
+        # Keep only structurally complete representations
+        if not current_premarket_session.empty and not current_daytime_session.empty and not current_nighttime_session.empty:
+            
+            # Find extreme points
+            pre_hh_idx = current_premarket_session['high'].idxmax()
+            pre_ll_idx = current_premarket_session['low'].idxmin()
+            
+            day_hh_idx = current_daytime_session['high'].idxmax()
+            day_ll_idx = current_daytime_session['low'].idxmin()
+            
+            night_hh_idx = current_nighttime_session['high'].idxmax()
+            night_ll_idx = current_nighttime_session['low'].idxmin()
+            
+            extremes = {
+                'pre_hh': (current_premarket_session.loc[pre_hh_idx, 'rel_time_hours'], current_premarket_session.loc[pre_hh_idx, 'high']),
+                'pre_ll': (current_premarket_session.loc[pre_ll_idx, 'rel_time_hours'], current_premarket_session.loc[pre_ll_idx, 'low']),
+                'day_hh': (current_daytime_session.loc[day_hh_idx, 'rel_time_hours'], current_daytime_session.loc[day_hh_idx, 'high']),
+                'day_ll': (current_daytime_session.loc[day_ll_idx, 'rel_time_hours'], current_daytime_session.loc[day_ll_idx, 'low']),
+                'night_hh': (current_nighttime_session.loc[night_hh_idx, 'rel_time_hours'], current_nighttime_session.loc[night_hh_idx, 'high']),
+                'night_ll': (current_nighttime_session.loc[night_ll_idx, 'rel_time_hours'], current_nighttime_session.loc[night_ll_idx, 'low'])
+            }
+            
+            rep = MarketRepresentation(
+                current_time_gap,
+                current_premarket_session,
+                current_daytime_session,
+                current_nighttime_session,
+                extremes
+            )
+            representations.append(rep)
+            
+    print(f"[✔] Extraction complete in {time.time() - extract_start:.2f} seconds.")
+    print(f"[i] Found {len(representations):,} complete daily market representations.")
+    
+    if len(representations) == 0:
+        print("[!] No complete sessions found. Exiting.")
+        return
+
+    print("\n" + "="*55)
+    print("3. GENERATING VECTORIZED OVERLAY PLOT")
+    print("="*55)
+    plot_start = time.time()
+    
+    # Fast merge of validated data structures
+    valid_sessions_dfs = [rep.to_dataframe() for rep in representations]
+    df_aligned = pd.concat(valid_sessions_dfs, ignore_index=True)
+    
+    up = df_aligned['close'] >= df_aligned['open']
+    down = ~up
+    
+    # Render width parameters UPDATED FOR M3 (3 minutes)
+    width = (3.0 / 60.0) * 0.8  # 80% width of a 3-minute period mapped in hours
+    w = width / 2
+    
+    x = df_aligned['rel_time_hours'].values
+    o = df_aligned['open'].values
+    c = df_aligned['close'].values
+    h = df_aligned['high'].values
+    l = df_aligned['low'].values
+    
+    x_up, o_up, c_up, h_up, l_up = x[up], o[up], c[up], h[up], l[up]
+    x_down, o_down, c_down, h_down, l_down = x[down], o[down], c[down], h[down], l[down]
+    
+    # Memory-efficient vectorized polygon coordinate mapping
+    wicks_up = np.empty((len(x_up), 2, 2))
+    wicks_up[:, 0, 0] = x_up; wicks_up[:, 0, 1] = l_up
+    wicks_up[:, 1, 0] = x_up; wicks_up[:, 1, 1] = h_up
+    
+    wicks_down = np.empty((len(x_down), 2, 2))
+    wicks_down[:, 0, 0] = x_down; wicks_down[:, 0, 1] = l_down
+    wicks_down[:, 1, 0] = x_down; wicks_down[:, 1, 1] = h_down
+    
+    boxes_up = np.empty((len(x_up), 4, 2))
+    boxes_up[:, 0, 0] = x_up - w; boxes_up[:, 0, 1] = o_up
+    boxes_up[:, 1, 0] = x_up + w; boxes_up[:, 1, 1] = o_up
+    boxes_up[:, 2, 0] = x_up + w; boxes_up[:, 2, 1] = c_up
+    boxes_up[:, 3, 0] = x_up - w; boxes_up[:, 3, 1] = c_up
+    
+    boxes_down = np.empty((len(x_down), 4, 2))
+    boxes_down[:, 0, 0] = x_down - w; boxes_down[:, 0, 1] = o_down
+    boxes_down[:, 1, 0] = x_down + w; boxes_down[:, 1, 1] = o_down
+    boxes_down[:, 2, 0] = x_down + w; boxes_down[:, 2, 1] = c_down
+    boxes_down[:, 3, 0] = x_down - w; boxes_down[:, 3, 1] = c_down
+    
+    # White-themed rendering (Massive Canvas for 1200 DPI Detail)
+    plt.style.use('default')
+    fig, ax = plt.subplots(figsize=(30, 15), facecolor='white')
+    ax.set_facecolor('white')
+    
+    up_color = '#26a69a'
+    down_color = '#ef5350'
+    alpha_val = 0.1  # Exactly 10% opacity
+    
+    # Hardware-accelerated GPU/CPU rendering using Collections
+    lc_up = LineCollection(wicks_up, colors=up_color, linewidths=0.5, alpha=alpha_val)
+    lc_down = LineCollection(wicks_down, colors=down_color, linewidths=0.5, alpha=alpha_val)
+    pc_up = PolyCollection(boxes_up, facecolors=up_color, edgecolors='none', alpha=alpha_val)
+    pc_down = PolyCollection(boxes_down, facecolors=down_color, edgecolors='none', alpha=alpha_val)
+    
+    ax.add_collection(lc_up)
+    ax.add_collection(lc_down)
+    ax.add_collection(pc_up)
+    ax.add_collection(pc_down)
+    
+    # Plot extreme points
+    hh_x, hh_y = [], []
+    ll_x, ll_y = [], []
+    
+    for rep in representations:
+        if rep.extremes:
+            ext = rep.extremes
+            hh_x.extend([ext['pre_hh'][0], ext['day_hh'][0], ext['night_hh'][0]])
+            hh_y.extend([ext['pre_hh'][1], ext['day_hh'][1], ext['night_hh'][1]])
+            
+            ll_x.extend([ext['pre_ll'][0], ext['day_ll'][0], ext['night_ll'][0]])
+            ll_y.extend([ext['pre_ll'][1], ext['day_ll'][1], ext['night_ll'][1]])
+            
+    ax.scatter(hh_x, hh_y, color='blue', s=15, alpha=0.6, label='Highest Highs', zorder=5)
+    ax.scatter(ll_x, ll_y, color='red', s=15, alpha=0.6, label='Lowest Lows', zorder=5)
+    
+    ax.autoscale_view()
+    
+    # Annotations & Formatting
+    ax.set_title("Complete Daily Market Representation Overlay (M3 Timeframe)\nAligned at 0:00 UTC", fontsize=20, fontweight='bold', pad=15)
+    ax.set_xlabel("Hours Relative to 0:00 UTC", fontsize=16, labelpad=10)
+    ax.set_ylabel("Price Deviation from 0:00 UTC Open", fontsize=16, labelpad=10)
+    
+    min_x, max_x = ax.get_xlim()
+    ax.set_xticks(np.arange(np.floor(min_x), np.ceil(max_x)+1, 2))
+    
+    # Segment shading
+    ax.axvspan(min_x, 0, color='#e3f2fd', alpha=0.3, label='Premarket (< 0:00)')
+    ax.axvspan(0, 12, color='#fff3e0', alpha=0.3, label='Daytime (0:00 - 12:00)')
+    ax.axvspan(12, max_x, color='#f3e5f5', alpha=0.3, label='Nighttime (> 12:00)')
+    
+    ax.grid(True, linestyle='--', alpha=0.5)
+    ax.axvline(0, color='black', linestyle='-', linewidth=2.0, alpha=0.8, label='0:00 UTC Alignment')
+    ax.axhline(0, color='black', linestyle='-', linewidth=2.0, alpha=0.8)
+    
+    ax.legend(loc='upper left', fontsize=14)
+    plt.tight_layout()
+    
+    # High Resolution Export Setup
+    export_path = 'market_representation_M3_high_res.png'
+    print(f"[✔] Exporting Extreme-Res image (1200 DPI) to {export_path}...")
+    plt.savefig(export_path, dpi=1200, bbox_inches='tight')
+    print(f"[✔] Plot generated and saved in {time.time() - plot_start:.2f} seconds.")
+    print("="*55)
+    plt.show()
+
+# Execution Call
+# Upload your M3 file to Colab and update the path accordingly.
+csv_path = '/content/XAUUSDc_M3_2014-01-14_to_2026-04-29_1060828records_88092KB.csv'
+run_market_representation_pipeline(csv_path)