perforatedai.tracker_perforatedai
1# Copyright (c) 2025 Perforated AI 2 3import io 4import math 5import os 6import shutil 7import sys 8import time 9from datetime import datetime 10from pydoc import locate 11 12import matplotlib as mpl 13import matplotlib.pyplot as plt 14import numpy as np 15import pandas as pd 16import torch 17import torch.nn as nn 18import torch.nn.functional as F 19import torch.nn.init as init 20import pdb 21 22from perforatedai import globals_perforatedai as GPA 23from perforatedai import modules_perforatedai as PA 24from perforatedai import utils_perforatedai as UPA 25 26try: 27 from dashboard_utils.event_emitter import emitter as _dashboard_emitter 28except ImportError as e: 29 print(f"[ERROR] Failed to import dashboard_utils.event_emitter: {e}") 30 _dashboard_emitter = None 31 32 33def _pai_log(level, message): 34 if _dashboard_emitter is not None: 35 _dashboard_emitter.log(GPA.pc, level, message) 36 else: 37 if level in ("warning", "error") or not GPA.pc.get_silent(): 38 print(message) 39 40 41try: 42 from perforatedbp import tracker_pbp as TPB 43except ModuleNotFoundError: 44 pass # Module not found, pass silently 45except ImportError as e: 46 print(f"Import error occurred: {e}") 47 48mpl.use("Agg") 49 50# Status constants for restructuring during add_validation_score 51NO_MODEL_UPDATE = 0 52NETWORK_RESTRUCTURED = 1 53TRAINING_COMPLETE = 2 54 55# Status constant for each batch 56STEP_CLEARED = 0 57STEP_CALLED = 1 58 59# Shared palette for save_graphs PNGs so the local graphs read as the same 60# product as the dashboard charts. White background is kept intentionally - 61# these PNGs get embedded in docs and papers. 62_PAI_PALETTE = { 63 "dendrite": "#00E0B5", # Dendrite / PAI 64 "normal": "#FD4D00", # Normal 65 "validation": "#59CD6C", # Validation score 66 "train": "#E072A4", # Train score 67 "test": "#9781E6", # Test / next extra score 68 "learning_rate": "#1A659E", # Learning rate 69 "grid": "#004145", # Grid lines (use at low alpha) 70 "marker_best": "#1A1A1A", # Global-best marker (not a data colour) 71 "marker_improved": "#7A7A7A", # Epoch-last-improved marker 72} 73 74# Cycle for extra_scores beyond train/test, and for dendrite layer curves. 75_PAI_EXTRA_CYCLE = ["#9781E6", "#1A659E", "#00A5A5", "#F2B807"] 76_PAI_DENDRITE_CYCLE = [ 77 "#00E0B5", 78 "#00A5A5", 79 "#1A659E", 80 "#9781E6", 81 "#E072A4", 82 "#F2B807", 83] 84 85 86def _pai_grid(ax): 87 """Apply the shared PAI grid styling to an axis.""" 88 ax.grid(True, color=_PAI_PALETTE["grid"], alpha=0.25) 89 90 91def _pai_extra_score_color(extra_name, extra_index): 92 """Colour for an ``extra_scores`` series by name, then by cycle position.""" 93 lowered = str(extra_name).lower() 94 if lowered == "train": 95 return _PAI_PALETTE["train"] 96 if lowered == "test": 97 return _PAI_PALETTE["test"] 98 return _PAI_EXTRA_CYCLE[extra_index % len(_PAI_EXTRA_CYCLE)] 99 100 101def update_restructuring_status(old_status, new_status): 102 """Update restructured variable during add_validation_score 103 104 Update the restructuring status based on the new status. 105 If the new status is that there was not an update, 106 dont overwrite the old status which may show there was an update. 107 108 Parameters 109 ---------- 110 old_status : int 111 The old restructuring status. 112 new_status : int 113 The new restructuring status. 114 115 Returns 116 ------- 117 int 118 The updated restructuring status. 119 120 """ 121 if new_status == NETWORK_RESTRUCTURED or new_status == TRAINING_COMPLETE: 122 return NETWORK_RESTRUCTURED 123 else: 124 return old_status 125 126 127def update_learning_rate(): 128 """Update the learning rate in the tracker. 129 130 Parameters 131 ---------- 132 None 133 134 Returns 135 ------- 136 None 137 This function does not return a value. 138 """ 139 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 140 learning_rate = param_group["lr"] 141 GPA.pai_tracker.add_learning_rate(learning_rate) 142 143 144def update_param_count(net): 145 """Update the parameter count in the tracker if not already set. 146 147 Parameters 148 ---------- 149 net : torch.nn.Module 150 The neural network model to count parameters for. 151 Returns 152 ------- 153 None 154 """ 155 if len(GPA.pai_tracker.member_vars["param_counts"]) == 0: 156 GPA.pai_tracker.member_vars["param_counts"].append(UPA.count_params(net)) 157 158 159def check_input_problems(net, accuracy): 160 """Check for potential input problems in add_validation_score. 161 162 Parameters 163 ---------- 164 net : torch.nn.Module 165 The neural network model to check. 166 accuracy : float, int, or torch.Tensor 167 The accuracy score to validate. 168 169 Returns 170 ------- 171 float 172 The validated accuracy score. 173 174 """ 175 176 # Make sure you are passing in the model and not the dataparallel wrapper 177 if issubclass(type(net), nn.DataParallel): 178 _pai_log("error", "Need to call .module when using add validation score") 179 pdb.set_trace() 180 sys.exit(-1) 181 182 if "module" in net.__dir__(): 183 _pai_log("error", "Need to call .module when using add validation score") 184 pdb.set_trace() 185 sys.exit(-1) 186 187 if not isinstance(accuracy, (float, int)): 188 try: 189 accuracy = accuracy.item() 190 except: 191 _pai_log( 192 "error", 193 f"Scores added for add_validation_score should be float, int, or tensor, yours is a: {type(accuracy)}", 194 ) 195 pdb.set_trace() 196 sys.exit(-1) 197 return accuracy 198 199 200def update_running_accuracy(accuracy, epochs_since_cycle_switch): 201 """Add the new accuracy to the tracker. 202 203 Parameters 204 ---------- 205 accuracy : float, int, or torch.Tensor 206 The accuracy score to add. 207 epochs_since_cycle_switch : int 208 The number of epochs since the last cycle switch. 209 210 Returns 211 ------- 212 None 213 214 """ 215 # Only update running_accuracy when neurons are being updated 216 if GPA.pai_tracker.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 217 if epochs_since_cycle_switch < GPA.pc.get_initial_history_after_switches(): 218 if epochs_since_cycle_switch <= 0: 219 GPA.pai_tracker.member_vars["running_accuracy"] = accuracy 220 else: 221 GPA.pai_tracker.member_vars[ 222 "running_accuracy" 223 ] = GPA.pai_tracker.member_vars["running_accuracy"] * ( 224 1 - (1.0 / (epochs_since_cycle_switch + 1)) 225 ) + accuracy * ( 226 1.0 / (epochs_since_cycle_switch + 1) 227 ) 228 else: 229 GPA.pai_tracker.member_vars[ 230 "running_accuracy" 231 ] = GPA.pai_tracker.member_vars["running_accuracy"] * ( 232 1.0 - 1.0 / GPA.pc.get_history_lookback() 233 ) + accuracy * ( 234 1.0 / GPA.pc.get_history_lookback() 235 ) 236 237 GPA.pai_tracker.member_vars["accuracies"].append(accuracy) 238 if GPA.pai_tracker.member_vars["mode"] == "n": 239 GPA.pai_tracker.member_vars["n_accuracies"].append(accuracy) 240 241 if ( 242 GPA.pc.get_drawing_pai() 243 or GPA.pai_tracker.member_vars["mode"] == "n" 244 or GPA.pc.get_learn_dendrites_live() 245 ): 246 GPA.pai_tracker.member_vars["running_accuracies"].append( 247 GPA.pai_tracker.member_vars["running_accuracy"] 248 ) 249 250 251def score_beats_current_best(new_score, old_score): 252 """Check if the new score beats the current best score. 253 254 Parameters 255 ---------- 256 new_score : float 257 The new score to compare. 258 old_score : float 259 The old score to compare against. 260 261 Returns 262 ------- 263 bool 264 True if the new score beats the old score, False otherwise. 265 266 Notes 267 ----- 268 Must beat the old score by the margins set in globals for improvement thresholds. 269 270 """ 271 return ( 272 GPA.pai_tracker.member_vars["maximizing_score"] 273 and (new_score * (1.0 - GPA.pc.get_improvement_threshold()) > old_score) 274 and new_score - GPA.pc.get_improvement_threshold_raw() > old_score 275 ) or ( 276 (not GPA.pai_tracker.member_vars["maximizing_score"]) 277 and (new_score * (1.0 + GPA.pc.get_improvement_threshold()) < old_score) 278 and (new_score + GPA.pc.get_improvement_threshold_raw()) < old_score 279 ) 280 281 282def check_new_best(net, accuracy, epochs_since_cycle_switch): 283 """Check if the new accuracy is a new best. 284 285 Performs saves if new best score is found. 286 287 Parameters 288 ---------- 289 net : torch.nn.Module 290 The neural network model being trained. 291 accuracy : float 292 The accuracy score to check. 293 epochs_since_cycle_switch : int 294 The number of epochs since the last cycle switch. 295 296 Returns 297 ------- 298 None 299 300 """ 301 score_improved = score_beats_current_best( 302 GPA.pai_tracker.member_vars["running_accuracy"], 303 GPA.pai_tracker.member_vars["current_best_validation_score"], 304 ) 305 306 enough_time = ( 307 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 308 ) or (GPA.pai_tracker.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME) 309 310 if ( 311 score_improved 312 or GPA.pai_tracker.member_vars["current_best_validation_score"] == 0 313 ) and enough_time: 314 315 if GPA.pai_tracker.member_vars["maximizing_score"]: 316 if GPA.pc.get_verbose(): 317 print( 318 f"\n\nGot score of {accuracy:.10f} " 319 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 320 f"*{1-GPA.pc.get_improvement_threshold()}=" 321 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 - GPA.pc.get_improvement_threshold())}) ' 322 f'which is higher than {GPA.pai_tracker.member_vars["current_best_validation_score"]:.10f} ' 323 f"by {GPA.pc.get_improvement_threshold_raw()} so setting epoch to " 324 f'{GPA.pai_tracker.member_vars["num_epochs_run"]}\n\n' 325 ) 326 else: 327 if GPA.pc.get_verbose(): 328 print( 329 f"\n\nGot score of {accuracy:.10f} " 330 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 331 f"*{1+GPA.pc.get_improvement_threshold()}=" 332 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 + GPA.pc.get_improvement_threshold())}) ' 333 f'which is lower than {GPA.pai_tracker.member_vars["current_best_validation_score"]:.10f} ' 334 f'so setting epoch to {GPA.pai_tracker.member_vars["num_epochs_run"]}\n\n' 335 ) 336 337 # Set the new best score 338 GPA.pai_tracker.member_vars["current_best_validation_score"] = ( 339 GPA.pai_tracker.member_vars["running_accuracy"] 340 ) 341 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 342 GPA.pai_tracker.member_vars["num_epochs_run"] 343 ) 344 if GPA.pc.get_verbose(): 345 print( 346 f'2 epoch improved is {GPA.pai_tracker.member_vars["epoch_last_improved"]}' 347 ) 348 # Immediately update this list before saving so loading will have it correctly 349 GPA.pai_tracker.member_vars["last_improved_accuracies"].append( 350 GPA.pai_tracker.member_vars["epoch_last_improved"] 351 ) 352 # Check if global best 353 is_global_best = score_beats_current_best( 354 GPA.pai_tracker.member_vars["current_best_validation_score"], 355 GPA.pai_tracker.member_vars["global_best_validation_score"], 356 ) 357 358 if ( 359 is_global_best 360 or GPA.pai_tracker.member_vars["global_best_validation_score"] == 0 361 ): 362 if GPA.pc.get_verbose(): 363 print( 364 f"This also beats global best of " 365 f'{GPA.pai_tracker.member_vars["global_best_validation_score"]} so saving' 366 ) 367 GPA.pai_tracker.member_vars["global_best_validation_score"] = ( 368 GPA.pai_tracker.member_vars["current_best_validation_score"] 369 ) 370 GPA.pai_tracker.member_vars["current_n_set_global_best"] = True 371 UPA.save_system(net, GPA.pc.get_save_name(), "best_model") 372 if GPA.pc.get_pai_saves(): 373 UPA.pai_save_system(net, GPA.pc.get_save_name(), "best_model") 374 else: 375 if GPA.pc.get_verbose(): 376 print("Not saving new best because:") 377 if epochs_since_cycle_switch <= GPA.pc.get_initial_history_after_switches(): 378 print( 379 f"Not enough history since switch {epochs_since_cycle_switch} <= " 380 f"{GPA.pc.get_initial_history_after_switches()}" 381 ) 382 elif GPA.pai_tracker.member_vars["maximizing_score"]: 383 print( 384 f"Got score of {accuracy} " 385 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 386 f"*{1-GPA.pc.get_improvement_threshold()}=" 387 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 - GPA.pc.get_improvement_threshold())}) ' 388 f"which is not higher than " 389 f'{GPA.pai_tracker.member_vars["current_best_validation_score"]}' 390 ) 391 else: 392 print( 393 f"Got score of {accuracy} " 394 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 395 f"*{1+GPA.pc.get_improvement_threshold()}=" 396 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 + GPA.pc.get_improvement_threshold())}) ' 397 f"which is not lower than " 398 f'{GPA.pai_tracker.member_vars["current_best_validation_score"]}' 399 ) 400 GPA.pai_tracker.member_vars["last_improved_accuracies"].append( 401 GPA.pai_tracker.member_vars["epoch_last_improved"] 402 ) 403 # If it's the first epoch, save as best anyway 404 if len(GPA.pai_tracker.member_vars["accuracies"]) == 1: 405 if GPA.pc.get_verbose(): 406 print("Saving first model or all models") 407 UPA.save_system(net, GPA.pc.get_save_name(), "best_model") 408 if GPA.pc.get_pai_saves(): 409 UPA.pai_save_system(net, GPA.pc.get_save_name(), "best_model") 410 411 412def process_no_improvement(net): 413 """Handle the case where no improvement is observed. 414 415 If the new dendrite did not improve scores, but its time to switch modes 416 either trigger the end of learning or reset to the previous dendrite 417 to try again. 418 419 Parameters 420 ---------- 421 net : torch.nn.Module 422 The neural network model being trained. 423 424 Returns 425 ------- 426 int 427 The status of restructuring or training completion. 428 torch.nn.Module 429 The potentially modified neural network model. 430 431 """ 432 if GPA.pc.get_verbose(): 433 print( 434 f"Planning to switch to p mode but best beat last: " 435 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]} ' 436 f"current start lr steps: " 437 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 438 f"and last maximum lr steps: " 439 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 440 f'for rate: {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]:.8f}' 441 ) 442 443 now = datetime.now() 444 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 445 446 if GPA.pc.get_verbose(): 447 print( 448 f'1 saving break {dt_string}_noImprove_lr_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 449 ) 450 451 GPA.pai_tracker.save_graphs( 452 f'{dt_string}_noImprove_lr_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 453 ) 454 455 if ( 456 GPA.pai_tracker.member_vars["num_dendrite_tries"] 457 < GPA.pc.get_max_dendrite_tries() -1 458 ): 459 _pai_log( 460 "info", 461 f"The newest added dendrites did not improve but current tries " 462 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"] + 1} ' 463 f"is less than max tries {GPA.pc.get_max_dendrite_tries()} " 464 f"so loading last switch and trying new Dendrites.", 465 ) 466 old_tries = GPA.pai_tracker.member_vars["num_dendrite_tries"] 467 # Load best model from previous n mode 468 net = UPA.change_learning_modes( 469 net, 470 GPA.pc.get_save_name(), 471 "best_model", 472 GPA.pai_tracker.member_vars["doing_pai"], 473 ) 474 GPA.pai_tracker.member_vars["num_dendrite_tries"] = old_tries + 1 475 return NETWORK_RESTRUCTURED, net 476 else: 477 _pai_log( 478 "info", 479 f"The newest added dendrites did not improve system and " 480 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"] + 1} > ' 481 f"{GPA.pc.get_max_dendrite_tries()} so returning training_complete.", 482 ) 483 _pai_log("info", "You should now exit your training loop and best_model will be your final model for inference") 484 if not GPA.pc.get_perforated_backpropagation() and GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 485 _pai_log("info", "For improved results, try perforated backpropagation next time!") 486 487 old_silent = GPA.pc.get_silent() 488 GPA.pc.set_silent(True) 489 UPA.load_system(net, GPA.pc.get_save_name(), "best_model", switch_call=True) 490 GPA.pc.set_silent(old_silent) 491 GPA.pai_tracker.save_graphs() 492 UPA.pai_save_system(net, GPA.pc.get_save_name(), "final_clean") 493 return TRAINING_COMPLETE, net 494 495 496def process_final_network(net): 497 """When the max number of dendrites has been hit load the best_model and return 498 499 Parameters 500 ---------- 501 net : torch.nn.Module 502 The neural network model being trained. 503 504 Returns 505 ------- 506 torch.nn.Module 507 The final neural network model. 508 """ 509 510 _pai_log("info", f"Last Dendrites were good and this hit the max of {GPA.pc.get_max_dendrites()}") 511 if not GPA.pc.get_perforated_backpropagation() and GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 512 _pai_log("info", "For improved results, try perforated backpropagation next time!") 513 GPA.pai_tracker.save_graphs("before_final") 514 UPA.load_system(net, GPA.pc.get_save_name(), "best_model", switch_call=True) 515 GPA.pai_tracker.save_graphs() 516 UPA.pai_save_system(net, GPA.pc.get_save_name(), "final_clean") 517 return net 518 519 520def process_scheduler_update(net, accuracy, epochs_since_cycle_switch): 521 """Updates the scheduler 522 523 This increments the scheduler, but if we are automatically sweeping 524 to find the best initial learning rate for a new set of dendrites 525 this function also triggers the network at addition time to 526 try the next value. 527 528 Process for finding best initial learning rate for dendrites: 529 1. Start at default rate 530 2. Learn at that rate until scheduler increments twice 531 3. Save that version, start dendrites at LR current increment - 1 532 4. Repeat 2 and 3 until version has worse final score at set LR 533 5. Load previous model with best accuracy at that LR as initial rate 534 535 Parameters 536 ---------- 537 net : torch.nn.Module 538 The neural network model being trained. 539 accuracy : float 540 The accuracy of the model at the current learning rate. 541 epochs_since_cycle_switch : int 542 The number of epochs since the last cycle switch. 543 544 Returns 545 ------- 546 int 547 The status of restructuring or training completion. 548 torch.nn.Module 549 The potentially modified neural network model. 550 """ 551 552 restructured = False 553 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 554 learning_rate1 = param_group["lr"] 555 556 if ( 557 type(GPA.pai_tracker.member_vars["scheduler_instance"]) 558 is torch.optim.lr_scheduler.ReduceLROnPlateau 559 ): 560 if ( 561 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 562 or GPA.pai_tracker.member_vars["mode"] == "p" 563 ): 564 if GPA.pc.get_verbose(): 565 print( 566 f"Updating scheduler with last improved " 567 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]} ' 568 f'from current {GPA.pai_tracker.member_vars["num_epochs_run"]}' 569 ) 570 if GPA.pai_tracker.member_vars["scheduler"] is not None: 571 GPA.pai_tracker.member_vars["scheduler_instance"].step(metrics=accuracy) 572 if ( 573 GPA.pai_tracker.member_vars["scheduler"] 574 is torch.optim.lr_scheduler.ReduceLROnPlateau 575 ): 576 if GPA.pc.get_verbose(): 577 print( 578 f"Scheduler is now at " 579 f'{GPA.pai_tracker.member_vars["scheduler_instance"].num_bad_epochs} bad epochs' 580 ) 581 else: 582 if GPA.pc.get_verbose(): 583 print("Not stepping optimizer since hasnt initialized") 584 585 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 586 if ( 587 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 588 or GPA.pai_tracker.member_vars["mode"] == "p" 589 ): 590 if GPA.pc.get_verbose(): 591 if hasattr(GPA.pai_tracker.member_vars["scheduler_instance"], '_step_count'): 592 count = GPA.pai_tracker.member_vars["scheduler_instance"]._step_count 593 else: 594 count = GPA.pai_tracker.member_vars["scheduler_instance"].last_epoch 595 596 print( 597 f"Incrementing scheduler to count " 598 f'{count}' 599 ) 600 GPA.pai_tracker.member_vars["scheduler_instance"].step() 601 if ( 602 GPA.pai_tracker.member_vars["scheduler"] 603 is torch.optim.lr_scheduler.ReduceLROnPlateau 604 ): 605 if GPA.pc.get_verbose(): 606 print( 607 f"Scheduler is now at " 608 f'{GPA.pai_tracker.member_vars["scheduler_instance"].num_bad_epochs} bad epochs' 609 ) 610 611 if ( 612 epochs_since_cycle_switch <= GPA.pc.get_initial_history_after_switches() 613 and GPA.pai_tracker.member_vars["mode"] == "n" 614 ): 615 if GPA.pc.get_verbose(): 616 print( 617 f"Not stepping with history {GPA.pc.get_initial_history_after_switches()} " 618 f"and current {epochs_since_cycle_switch}" 619 ) 620 621 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 622 learning_rate2 = param_group["lr"] 623 624 stepped = False 625 at_last_count = False 626 627 if GPA.pc.get_verbose(): 628 print( 629 f"Checking if at last with scores " 630 f'{len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"])}, ' 631 f"count since switch {epochs_since_cycle_switch} " 632 f"and last total lr step count " 633 f'{GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"]}' 634 ) 635 636 # Check if at double or exactly the test count 637 if ( 638 len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 0 639 and epochs_since_cycle_switch 640 == GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] * 2 641 ) or ( 642 len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 1 643 and epochs_since_cycle_switch 644 == GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] 645 ): 646 at_last_count = True 647 648 if GPA.pc.get_verbose(): 649 print( 650 f"At last count {at_last_count} with count {epochs_since_cycle_switch} " 651 f'and last LR count {GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"]}' 652 ) 653 654 if learning_rate1 != learning_rate2: 655 stepped = True 656 GPA.pai_tracker.member_vars["current_step_count"] += 1 657 658 if GPA.pc.get_verbose(): 659 print( 660 f"Learning rate just stepped to {learning_rate2:.10e} " 661 f'with {GPA.pai_tracker.member_vars["current_step_count"]} total steps' 662 ) 663 664 if ( 665 GPA.pai_tracker.member_vars["current_step_count"] 666 == GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] 667 ): 668 if GPA.pc.get_verbose(): 669 print( 670 f'{GPA.pai_tracker.member_vars["current_step_count"]} ' 671 f"steps is the max of the last switch mode" 672 ) 673 # Set it when 1->2 gets to 2, not when 0->1 hits 2 as stopping point 674 if ( 675 GPA.pai_tracker.member_vars["current_step_count"] 676 - GPA.pai_tracker.member_vars[ 677 "current_n_learning_rate_initial_skip_steps" 678 ] 679 == 1 680 ): 681 GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] = ( 682 epochs_since_cycle_switch 683 ) 684 685 if GPA.pc.get_verbose(): 686 print( 687 f"Learning rates were {learning_rate1:.8e} and {learning_rate2:.8e} " 688 f'started with {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 689 f'and is now at {GPA.pai_tracker.member_vars["current_step_count"]} ' 690 f'committed {GPA.pai_tracker.member_vars["committed_to_initial_rate"]} ' 691 f"then either this (non zero) or eventually comparing to " 692 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 693 f'steps or rate {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]:.8f}' 694 ) 695 696 # If learning rate just stepped, check restart at lower rate 697 if ( 698 (GPA.pai_tracker.member_vars["scheduler"] is not None) 699 and 700 # If potentially might have higher accuracy 701 ( 702 (GPA.pai_tracker.member_vars["mode"] == "n") 703 or GPA.pc.get_learn_dendrites_live() 704 ) 705 and 706 # And learning rate just stepped 707 (stepped or at_last_count) 708 ): 709 710 # If this is the first dendrite addition (last_max_learning_rate_steps == 0), 711 # immediately commit to the initial rate without searching 712 if GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] == 0: 713 if GPA.pc.get_verbose(): 714 print( 715 f"First dendrite addition detected (last_max_learning_rate_steps == 0), " 716 f"immediately committing to initial rate without search" 717 ) 718 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 719 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 720 GPA.pai_tracker.member_vars["current_step_count"] 721 ) 722 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 723 learning_rate2 724 ) 725 726 # If hasn't committed to a learning rate for this cycle yet 727 if not GPA.pai_tracker.member_vars["committed_to_initial_rate"]: 728 best_score_so_far = GPA.pai_tracker.member_vars[ 729 "global_best_validation_score" 730 ] 731 732 if GPA.pc.get_verbose(): 733 print( 734 f"In statements to check next learning rate with " 735 f"stepped {stepped} and max count {at_last_count}" 736 ) 737 738 # If no scores saved for this dendrite and initial LR test did second step 739 if len( 740 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] 741 ) == 0 and ( 742 GPA.pai_tracker.member_vars["current_step_count"] 743 - GPA.pai_tracker.member_vars[ 744 "current_n_learning_rate_initial_skip_steps" 745 ] 746 == 2 747 or at_last_count 748 ): 749 750 restructured = True 751 GPA.pai_tracker.clear_optimizer_and_scheduler() 752 753 # Save system for this initial condition 754 old_global = GPA.pai_tracker.member_vars["global_best_validation_score"] 755 old_accuracy = GPA.pai_tracker.member_vars[ 756 "current_best_validation_score" 757 ] 758 old_counts = GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] 759 skip1 = GPA.pai_tracker.member_vars[ 760 "current_n_learning_rate_initial_skip_steps" 761 ] 762 763 now = datetime.now() 764 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 765 766 GPA.pai_tracker.save_graphs( 767 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 768 ) 769 770 if GPA.pc.get_test_saves(): 771 UPA.save_system( 772 net, 773 GPA.pc.get_save_name(), 774 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 775 ) 776 777 if GPA.pc.get_verbose(): 778 print( 779 f"Saving with initial steps: {dt_string}_PBCount_" 780 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 781 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 782 f"with current best {old_accuracy}" 783 ) 784 785 # Load back at start and try with lower initial learning rate 786 net = UPA.load_system( 787 net, 788 GPA.pc.get_save_name(), 789 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 790 switch_call=True, 791 ) 792 GPA.pai_tracker.member_vars[ 793 "current_n_learning_rate_initial_skip_steps" 794 ] = (skip1 + 1) 795 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"].append( 796 old_accuracy 797 ) 798 GPA.pai_tracker.member_vars["global_best_validation_score"] = old_global 799 GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] = old_counts 800 801 # If there is one score already, this is first step at next score 802 elif len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 1: 803 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"].append( 804 GPA.pai_tracker.member_vars["current_best_validation_score"] 805 ) 806 807 # If this LR's score was worse than last LR's score 808 lr_score_worse = False 809 if GPA.pai_tracker.member_vars["maximizing_score"]: 810 lr_score_worse = ( 811 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0] 812 > GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1] 813 ) 814 else: 815 lr_score_worse = ( 816 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0] 817 < GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1] 818 ) 819 820 if lr_score_worse: 821 restructured = True 822 GPA.pai_tracker.clear_optimizer_and_scheduler() 823 824 if GPA.pc.get_verbose(): 825 print( 826 f'Got initial {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1} ' 827 f'step score {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0]} ' 828 f'and {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 829 f'score at step {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1]} ' 830 f"so loading old score" 831 ) 832 833 prior_best = GPA.pai_tracker.member_vars[ 834 "current_cycle_lr_max_scores" 835 ][0] 836 837 now = datetime.now() 838 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 839 840 GPA.pai_tracker.save_graphs( 841 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 842 ) 843 844 if GPA.pc.get_test_saves(): 845 UPA.save_system( 846 net, 847 GPA.pc.get_save_name(), 848 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 849 ) 850 851 if GPA.pc.get_verbose(): 852 print( 853 f"Saving with initial steps: {dt_string}_PBCount_" 854 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 855 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 856 ) 857 858 if GPA.pc.get_test_saves(): 859 net = UPA.load_system( 860 net, 861 GPA.pc.get_save_name(), 862 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1}', 863 switch_call=True, 864 ) 865 866 # Save graphs for chosen one 867 now = datetime.now() 868 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 869 870 GPA.pai_tracker.save_graphs( 871 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}PICKED' 872 ) 873 874 if GPA.pc.get_test_saves(): 875 UPA.save_system( 876 net, 877 GPA.pc.get_save_name(), 878 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 879 ) 880 881 if GPA.pc.get_verbose(): 882 print( 883 f"Saving with initial steps: {dt_string}_PBCount_" 884 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 885 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 886 ) 887 888 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 889 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 890 GPA.pai_tracker.member_vars["current_step_count"] 891 ) 892 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 893 learning_rate2 894 ) 895 GPA.pai_tracker.member_vars["current_best_validation_score"] = ( 896 prior_best 897 ) 898 899 if GPA.pc.get_verbose(): 900 print( 901 f"Setting last max steps to " 902 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 903 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 904 ) 905 906 else: # Current LR score is better 907 if GPA.pc.get_verbose(): 908 print( 909 f'Got initial {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1} ' 910 f'step score {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0]} ' 911 f'and {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 912 f'score at step {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1]} ' 913 f"so NOT loading old score and continuing with this score" 914 ) 915 916 if at_last_count: # If this is the last one, set it to be picked 917 restructured = True 918 GPA.pai_tracker.clear_optimizer_and_scheduler() 919 920 now = datetime.now() 921 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 922 923 GPA.pai_tracker.save_graphs( 924 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}PICKED' 925 ) 926 927 if GPA.pc.get_test_saves(): 928 UPA.save_system( 929 net, 930 GPA.pc.get_save_name(), 931 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 932 ) 933 934 if GPA.pc.get_verbose(): 935 print( 936 f"Saving with initial steps: {dt_string}_PBCount_" 937 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 938 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 939 ) 940 941 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 942 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 943 GPA.pai_tracker.member_vars["current_step_count"] 944 ) 945 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 946 learning_rate2 947 ) 948 949 if GPA.pc.get_verbose(): 950 print( 951 f"Setting last max steps to " 952 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 953 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 954 ) 955 956 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 957 958 elif len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 2: 959 print( 960 "Should never be here. Please let Perforated AI know if this happened." 961 ) 962 pdb.set_trace() 963 964 GPA.pai_tracker.member_vars["global_best_validation_score"] = ( 965 best_score_so_far 966 ) 967 968 else: 969 if GPA.pc.get_verbose(): 970 print( 971 f"Setting last max steps to " 972 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 973 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 974 ) 975 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] += 1 976 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = learning_rate2 977 if restructured: 978 return NETWORK_RESTRUCTURED, net 979 else: 980 return NO_MODEL_UPDATE, net 981 982 983class PAINeuronModuleTracker: 984 """ 985 Manager class that tracks all neuron layers and dendrite layers, 986 controls when new dendrites are added, and communicates signals to modules. 987 """ 988 989 def __init__( 990 self, 991 doing_pai, 992 save_name, 993 making_graphs=True, 994 param_vals_setting=-1, 995 values_per_train_epoch=-1, 996 values_per_val_epoch=-1, 997 ): 998 """Initialize the tracker 999 1000 Parameters 1001 ---------- 1002 doing_pai : bool 1003 Whether or not dendrites should be used. 1004 save_name : str 1005 The base name for saving models and graphs. 1006 making_graphs : bool, optional 1007 Whether or not to generate graphs, by default True. 1008 param_vals_setting : int, optional 1009 Parameter values setting, by default -1. 1010 values_per_train_epoch : int, optional 1011 The number of values to look back for graphing 1012 during training, by default -1 (all values). 1013 values_per_val_epoch : int, optional 1014 The number of values to look back for graphing 1015 during validation, by default -1 (all values). 1016 Returns 1017 ------- 1018 None 1019 """ 1020 1021 # Dict of member vars and their types for saving 1022 self.member_vars = {} 1023 self.member_var_types = {} 1024 1025 # Whether or not PAI will be running 1026 self.member_vars["doing_pai"] = doing_pai 1027 self.member_var_types["doing_pai"] = "bool" 1028 1029 # How many Dendrites have been added 1030 self.member_vars["num_dendrites_added"] = 0 1031 self.member_var_types["num_dendrites_added"] = "int" 1032 1033 # How many Dendrites have been successfully integrated, does not count currently training dendrites 1034 self.member_vars["num_dendrites_integrated"] = 0 1035 self.member_var_types["num_dendrites_integrated"] = "int" 1036 1037 # How many cycles have been run, *2 or *2+1 of the above 1038 self.member_vars["num_cycles"] = 0 1039 self.member_var_types["num_cycles"] = "int" 1040 1041 # Pointers to all neuron wrapped modules 1042 self.neuron_module_vector = [] 1043 1044 # Pointers to all non neuron modules for tracking 1045 self.tracked_neuron_module_vector = [] 1046 1047 # Neuron training or dendrite training mode 1048 self.member_vars["mode"] = "n" 1049 self.member_var_types["mode"] = "string" 1050 1051 # Number of epochs run excluding overwritten epochs 1052 self.member_vars["num_epochs_run"] = -1 1053 self.member_var_types["num_epochs_run"] = "int" 1054 1055 # Number including overwritten epochs 1056 self.member_vars["total_epochs_run"] = -1 1057 self.member_var_types["total_epochs_run"] = "int" 1058 1059 # Last epoch that validation/correlation score was improved 1060 self.member_vars["epoch_last_improved"] = 0 1061 self.member_var_types["epoch_last_improved"] = "int" 1062 1063 # Running validation accuracy 1064 self.member_vars["running_accuracy"] = 0 1065 self.member_var_types["running_accuracy"] = "float" 1066 1067 # True if maxing validation, False if minimizing Loss 1068 self.member_vars["maximizing_score"] = True 1069 self.member_var_types["maximizing_score"] = "bool" 1070 1071 # Mode for switching back and forth between learning modes 1072 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 1073 self.member_var_types["switch_mode"] = "int" 1074 1075 # Epoch of the last switch 1076 self.member_vars["last_switch"] = 0 1077 self.member_var_types["last_switch"] = "int" 1078 1079 # Highest validation score from current cycle 1080 self.member_vars["current_best_validation_score"] = 0 1081 self.member_var_types["current_best_validation_score"] = "float" 1082 1083 # Last epoch where the learning rate was updated 1084 self.member_vars["initial_lr_test_epoch_count"] = -1 1085 self.member_var_types["initial_lr_test_epoch_count"] = "int" 1086 1087 # Highest validation score of full run 1088 self.member_vars["global_best_validation_score"] = 0 1089 self.member_var_types["global_best_validation_score"] = "float" 1090 1091 # List of switch epochs 1092 self.member_vars["switch_epochs"] = [] 1093 self.member_var_types["switch_epochs"] = "int array" 1094 1095 # Parameter counts at each network structure 1096 self.member_vars["param_counts"] = [] 1097 self.member_var_types["param_counts"] = "int array" 1098 1099 # List of epochs where switch was made to neuron training 1100 self.member_vars["n_switch_epochs"] = [] 1101 self.member_var_types["n_switch_epochs"] = "int array" 1102 1103 # List of epochs where switch was made to dendrite training 1104 self.member_vars["p_switch_epochs"] = [] 1105 self.member_var_types["p_switch_epochs"] = "int array" 1106 1107 # List of validation accuracies 1108 self.member_vars["accuracies"] = [] 1109 self.member_var_types["accuracies"] = "float array" 1110 1111 # List of epochs where score improved for scheduler updates 1112 self.member_vars["last_improved_accuracies"] = [] 1113 self.member_var_types["last_improved_accuracies"] = "int array" 1114 1115 # List of test accuracy scores registered 1116 self.member_vars["test_accuracies"] = [] 1117 self.member_var_types["test_accuracies"] = "float array" 1118 1119 # List of accuracies registered during neuron training 1120 self.member_vars["n_accuracies"] = [] 1121 self.member_var_types["n_accuracies"] = "float array" 1122 1123 # List of accuracies registered during dendrite training 1124 self.member_vars["p_accuracies"] = [] 1125 self.member_var_types["p_accuracies"] = "float array" 1126 1127 # Running average accuracies from recent epochs 1128 self.member_vars["running_accuracies"] = [] 1129 self.member_var_types["running_accuracies"] = "float array" 1130 1131 # List of additional scores recorded 1132 self.member_vars["extra_scores"] = {} 1133 self.member_var_types["extra_scores"] = "float array dictionary" 1134 1135 # Extra scores not set to be graphed 1136 self.member_vars["extra_scores_without_graphing"] = {} 1137 self.member_var_types["extra_scores_without_graphing"] = ( 1138 "float array dictionary" 1139 ) 1140 1141 # List of test scores 1142 self.member_vars["test_scores"] = [] 1143 self.member_var_types["test_scores"] = "float array" 1144 1145 # Extra scores calculated during neuron training 1146 self.member_vars["n_extra_scores"] = {} 1147 self.member_var_types["n_extra_scores"] = "float array dictionary" 1148 1149 # List of training losses calculated 1150 self.member_vars["training_loss"] = [] 1151 self.member_var_types["training_loss"] = "float array" 1152 1153 # List of learning rates at each epoch 1154 self.member_vars["training_learning_rates"] = [] 1155 self.member_var_types["training_learning_rates"] = "float array" 1156 1157 # Best dendrite scores 1158 self.member_vars["best_scores"] = [] 1159 self.member_var_types["best_scores"] = "float array array" 1160 1161 # Current dendrite scores 1162 self.member_vars["current_scores"] = [] 1163 self.member_var_types["current_scores"] = "float array array" 1164 1165 # Times for neuron training epochs 1166 self.member_vars["n_epoch_times"] = [] 1167 self.member_var_types["n_epoch_times"] = "float array" 1168 1169 # Timing values 1170 self.member_vars["p_epoch_times"] = [] 1171 self.member_var_types["p_epoch_times"] = "float array" 1172 self.member_vars["n_train_times"] = [] 1173 self.member_var_types["n_train_times"] = "float array" 1174 self.member_vars["p_train_times"] = [] 1175 self.member_var_types["p_train_times"] = "float array" 1176 self.member_vars["n_val_times"] = [] 1177 self.member_var_types["n_val_times"] = "float array" 1178 self.member_vars["p_val_times"] = [] 1179 self.member_var_types["p_val_times"] = "float array" 1180 1181 # Setting for tracking timing 1182 self.member_vars["manual_train_switch"] = False 1183 self.member_var_types["manual_train_switch"] = "bool" 1184 1185 # Tracking scores overwritten when reloading best model 1186 self.member_vars["overwritten_extras"] = [] 1187 self.member_var_types["overwritten_extras"] = "float array dictionary array" 1188 self.member_vars["overwritten_vals"] = [] 1189 self.member_var_types["overwritten_vals"] = "float array array" 1190 self.member_vars["overwritten_epochs"] = 0 1191 self.member_var_types["overwritten_epochs"] = "int" 1192 1193 # Setting for determining scores 1194 self.member_vars["param_vals_setting"] = GPA.pc.get_param_vals_setting() 1195 self.member_var_types["param_vals_setting"] = "int" 1196 1197 # Optimizer and scheduler types and instances 1198 self.member_vars["optimizer"] = None 1199 self.member_var_types["optimizer"] = "type" 1200 self.member_vars["scheduler"] = None 1201 self.member_var_types["scheduler"] = "type" 1202 self.member_vars["optimizer_instance"] = None 1203 self.member_var_types["optimizer_instance"] = "empty array" 1204 self.member_vars["scheduler_instance"] = None 1205 self.member_var_types["scheduler_instance"] = "empty array" 1206 1207 # Flag for if the tracker was loaded 1208 self.loaded = False 1209 1210 # flag for 1211 self.member_vars["step_status"] = STEP_CLEARED 1212 self.member_var_types["step_status"] = "int" 1213 1214 1215 # Settings for tracking learning rates 1216 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 1217 self.member_var_types["current_n_learning_rate_initial_skip_steps"] = "int" 1218 self.member_vars["last_max_learning_rate_steps"] = 0 1219 self.member_var_types["last_max_learning_rate_steps"] = "int" 1220 self.member_vars["last_max_learning_rate_value"] = -1 1221 self.member_var_types["last_max_learning_rate_value"] = "float" 1222 self.member_vars["current_cycle_lr_max_scores"] = [] 1223 self.member_var_types["current_cycle_lr_max_scores"] = "float array" 1224 self.member_vars["current_step_count"] = 0 1225 self.member_var_types["current_step_count"] = "int" 1226 self.member_vars["committed_to_initial_rate"] = True 1227 self.member_var_types["committed_to_initial_rate"] = "bool" 1228 self.member_vars["best_mean_score_improved_this_epoch"] = 0 1229 self.member_var_types["best_mean_score_improved_this_epoch"] = "int" 1230 1231 # Flag for if current dendrite achieved highest global score 1232 self.member_vars["current_n_set_global_best"] = True 1233 self.member_var_types["current_n_set_global_best"] = "bool" 1234 1235 # Number of tries adding this dendrite count 1236 self.member_vars["num_dendrite_tries"] = 0 1237 self.member_var_types["num_dendrite_tries"] = "int" 1238 1239 # Count of batches per epoch 1240 self.values_per_train_epoch = values_per_train_epoch 1241 self.values_per_val_epoch = values_per_val_epoch 1242 1243 self.save_name = save_name 1244 self.making_graphs = making_graphs 1245 1246 self.start_time = time.time() 1247 self.saved_time = 0 1248 self.start_epoch(internal_call=True) 1249 1250 if GPA.pc.get_verbose(): 1251 print(f'Initializing with switch_mode {self.member_vars["switch_mode"]}') 1252 1253 def to_string(self): 1254 """Convert tracker values to string for saving with safetensors. 1255 1256 Parameters 1257 ---------- 1258 None 1259 1260 Returns 1261 ------- 1262 str 1263 Serialized tracker state suitable for storage in a safetensors field. 1264 """ 1265 1266 full_string = "" 1267 for var in self.member_vars: 1268 full_string += var + "," 1269 if self.member_vars[var] is None: 1270 full_string += "None" 1271 full_string += "\n" 1272 elif self.member_var_types[var] == "bool": 1273 full_string += str(self.member_vars[var]) 1274 full_string += "\n" 1275 elif self.member_var_types[var] in ("int", "float", "string"): 1276 full_string += str(self.member_vars[var]) 1277 full_string += "\n" 1278 elif self.member_var_types[var] == "type": 1279 name = ( 1280 self.member_vars[var].__module__ 1281 + "." 1282 + self.member_vars[var].__name__ 1283 ) 1284 full_string += str(self.member_vars[var]) 1285 full_string += "\n" 1286 elif self.member_var_types[var] == "empty array": 1287 full_string += "[]" 1288 full_string += "\n" 1289 elif self.member_var_types[var] in ("int array", "float array"): 1290 full_string += "\n" 1291 string = "" 1292 for val in self.member_vars[var]: 1293 string += str(val) + "," 1294 # Remove the last comma 1295 string = string[:-1] 1296 full_string += string 1297 full_string += "\n" 1298 elif self.member_var_types[var] == "float array dictionary array": 1299 full_string += "\n" 1300 for array in self.member_vars[var]: 1301 for key in array: 1302 string = key + "," 1303 for val in array[key]: 1304 string += str(val) + "," 1305 # Remove the last comma 1306 string = string[:-1] 1307 full_string += string 1308 full_string += "\n" 1309 full_string += "endkey" 1310 full_string += "\n" 1311 full_string += "endarray" 1312 full_string += "\n" 1313 elif self.member_var_types[var] == "float array dictionary": 1314 full_string += "\n" 1315 for key in self.member_vars[var]: 1316 string = key + "," 1317 for val in self.member_vars[var][key]: 1318 string += str(val) + "," 1319 # Remove the last comma 1320 string = string[:-1] 1321 full_string += string 1322 full_string += "\n" 1323 full_string += "end" 1324 full_string += "\n" 1325 elif self.member_var_types[var] == "float array array": 1326 full_string += "\n" 1327 for array in self.member_vars[var]: 1328 string = "" 1329 for val in array: 1330 string += str(val) + "," 1331 # Remove the last comma 1332 string = string[:-1] 1333 full_string += string 1334 full_string += "\n" 1335 full_string += "end" 1336 full_string += "\n" 1337 else: 1338 print("Did not find a member variable") 1339 pdb.set_trace() 1340 return full_string 1341 1342 def from_string(self, string): 1343 """Load tracker values from string. 1344 1345 Parameters 1346 ---------- 1347 string : str 1348 The string to load from. 1349 1350 Returns 1351 ------- 1352 None 1353 This function does not return a value. 1354 """ 1355 f = io.StringIO(string) 1356 while True: 1357 line = f.readline() 1358 if not line: 1359 break 1360 vals = line.split(",") 1361 var = vals[0] 1362 1363 if self.member_var_types[var] == "bool": 1364 val = vals[1][:-1] 1365 if val == "True": 1366 self.member_vars[var] = True 1367 elif val == "False": 1368 self.member_vars[var] = False 1369 elif val == "1": 1370 self.member_vars[var] = 1 1371 elif val == "0": 1372 self.member_vars[var] = 0 1373 else: 1374 print("Something went wrong with loading") 1375 pdb.set_trace() 1376 elif self.member_var_types[var] == "int": 1377 val = vals[1] 1378 self.member_vars[var] = int(val) 1379 elif self.member_var_types[var] == "float": 1380 val = vals[1] 1381 self.member_vars[var] = float(val) 1382 elif self.member_var_types[var] == "string": 1383 val = vals[1][:-1] 1384 self.member_vars[var] = val 1385 elif self.member_var_types[var] == "type": 1386 # Ignore loading types, tracker should have them set up 1387 continue 1388 elif self.member_var_types[var] == "empty array": 1389 val = vals[1] 1390 self.member_vars[var] = [] 1391 elif self.member_var_types[var] == "int array": 1392 vals = f.readline()[:-1].split(",") 1393 self.member_vars[var] = [] 1394 if vals[0] == "": 1395 continue 1396 for val in vals: 1397 self.member_vars[var].append(int(val)) 1398 elif self.member_var_types[var] == "float array": 1399 vals = f.readline()[:-1].split(",") 1400 self.member_vars[var] = [] 1401 if vals[0] == "": 1402 continue 1403 for val in vals: 1404 self.member_vars[var].append(float(val)) 1405 elif self.member_var_types[var] == "float array dictionary array": 1406 self.member_vars[var] = [] 1407 line2 = f.readline()[:-1] 1408 while line2 != "endarray": 1409 temp = {} 1410 while line2 != "endkey": 1411 vals = line2.split(",") 1412 name = vals[0] 1413 temp[name] = [] 1414 vals = vals[1:] 1415 for val in vals: 1416 temp[name].append(float(val)) 1417 line2 = f.readline()[:-1] 1418 self.member_vars[var].append(temp) 1419 line2 = f.readline()[:-1] 1420 elif self.member_var_types[var] == "float array dictionary": 1421 self.member_vars[var] = {} 1422 line2 = f.readline()[:-1] 1423 while line2 != "end": 1424 vals = line2.split(",") 1425 name = vals[0] 1426 self.member_vars[var][name] = [] 1427 vals = vals[1:] 1428 for val in vals: 1429 self.member_vars[var][name].append(float(val)) 1430 line2 = f.readline()[:-1] 1431 elif self.member_var_types[var] == "float array array": 1432 self.member_vars[var] = [] 1433 line2 = f.readline()[:-1] 1434 while line2 != "end": 1435 vals = line2.split(",") 1436 self.member_vars[var].append([]) 1437 if line2: 1438 for val in vals: 1439 self.member_vars[var][-1].append(float(val)) 1440 line2 = f.readline()[:-1] 1441 else: 1442 print("Did not find a member variable") 1443 1444 pdb.set_trace() 1445 1446 def from_string_debug(self, string): 1447 """Debug function to print tracker values from string without loading them. 1448 1449 Parameters 1450 ---------- 1451 string : str 1452 The string to debug load from. 1453 1454 Returns 1455 ------- 1456 None 1457 This function does not return a value. 1458 """ 1459 f = io.StringIO(string) 1460 print("=== DEBUGGING TRACKER VARIABLES ===") 1461 1462 while True: 1463 line = f.readline() 1464 if not line: 1465 break 1466 vals = line.split(",") 1467 var = vals[0] 1468 1469 print(f"\nVariable: {var}") 1470 print(f"Type: {self.member_var_types.get(var, 'UNKNOWN TYPE')}") 1471 print(f"Current value: {self.member_vars.get(var, 'NOT SET')}") 1472 1473 if self.member_var_types.get(var) == "bool": 1474 val = vals[1][:-1] 1475 print(f"Would set to: {val} -> {val == 'True'}") 1476 1477 elif self.member_var_types.get(var) == "int": 1478 val = vals[1] 1479 print(f"Would set to: {int(val)}") 1480 1481 elif self.member_var_types.get(var) == "float": 1482 val = vals[1] 1483 print(f"Would set to: {float(val)}") 1484 1485 elif self.member_var_types.get(var) == "string": 1486 val = vals[1][:-1] 1487 print(f"Would set to: '{val}'") 1488 1489 elif self.member_var_types.get(var) == "type": 1490 print("Would skip (type loading)") 1491 1492 elif self.member_var_types.get(var) == "empty array": 1493 val = vals[1] 1494 print(f"Would set to: [] (empty array)") 1495 1496 elif self.member_var_types.get(var) == "int array": 1497 vals_line = f.readline()[:-1].split(",") 1498 print(f"Would set to int array with {len(vals_line)} elements:") 1499 if vals_line[0] != "": 1500 print( 1501 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1502 ) 1503 else: 1504 print(" Empty array") 1505 1506 elif self.member_var_types.get(var) == "float array": 1507 vals_line = f.readline()[:-1].split(",") 1508 print(f"Would set to float array with {len(vals_line)} elements:") 1509 if vals_line[0] != "": 1510 print( 1511 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1512 ) 1513 else: 1514 print(" Empty array") 1515 1516 elif self.member_var_types.get(var) == "float array dictionary array": 1517 print("Would process float array dictionary array:") 1518 array_count = 0 1519 line2 = f.readline()[:-1] 1520 while line2 != "endarray": 1521 key_count = 0 1522 while line2 != "endkey": 1523 vals_dict = line2.split(",") 1524 name = vals_dict[0] 1525 print( 1526 f" Array {array_count}, Key '{name}': {len(vals_dict)-1} elements" 1527 ) 1528 key_count += 1 1529 line2 = f.readline()[:-1] 1530 print(f" Array {array_count} has {key_count} keys") 1531 array_count += 1 1532 line2 = f.readline()[:-1] 1533 print(f" Total arrays: {array_count}") 1534 1535 elif self.member_var_types.get(var) == "float array dictionary": 1536 print("Would process float array dictionary:") 1537 line2 = f.readline()[:-1] 1538 key_count = 0 1539 while line2 != "end": 1540 vals_dict = line2.split(",") 1541 name = vals_dict[0] 1542 print(f" Key '{name}': {len(vals_dict)-1} elements") 1543 key_count += 1 1544 line2 = f.readline()[:-1] 1545 print(f" Total keys: {key_count}") 1546 1547 elif self.member_var_types.get(var) == "float array array": 1548 print("Would process float array array:") 1549 line2 = f.readline()[:-1] 1550 array_count = 0 1551 while line2 != "end": 1552 if line2: 1553 vals_array = line2.split(",") 1554 print(f" Array {array_count}: {len(vals_array)} elements") 1555 else: 1556 print(f" Array {array_count}: empty") 1557 array_count += 1 1558 line2 = f.readline()[:-1] 1559 print(f" Total arrays: {array_count}") 1560 1561 else: 1562 print(f"UNKNOWN TYPE: {self.member_var_types.get(var, 'NOT FOUND')}") 1563 1564 print("\n=== END DEBUG ===") 1565 1566 def save_tracker_settings(self): 1567 """Save tracker settings for DistributedDataParallel use. 1568 1569 Saves settings in save_name/array_dims.csv 1570 1571 Parameters 1572 ---------- 1573 None 1574 Returns 1575 ------- 1576 None 1577 1578 ----- 1579 Instructions for use are in API customization.md 1580 """ 1581 if not os.path.isdir(self.save_name): 1582 os.makedirs(self.save_name) 1583 f = open(self.save_name + "/array_dims.csv", "w") 1584 for layer in self.neuron_module_vector: 1585 dv = layer.dendrite_module.dendrite_values[0] 1586 shape_str = ",".join(str(s) for s in dv.dendrite_storage_shape.tolist()) 1587 f.write(f"{layer.name},{shape_str}\n") 1588 f.close() 1589 if not GPA.pc.get_silent(): 1590 print("Tracker settings saved.") 1591 print("You may now delete save_tracker_settings") 1592 1593 def initialize_tracker_settings(self): 1594 """Initialize tracker settings from saved file. 1595 1596 This function loads tracker settings from a CSV file and applies them 1597 to the layers the tracker is managing. 1598 1599 Parameters 1600 ---------- 1601 None 1602 1603 Returns 1604 ------- 1605 None 1606 1607 """ 1608 1609 channels = {} 1610 if not os.path.exists(self.save_name + "/array_dims.csv"): 1611 print( 1612 "You must call save_tracker_settings before " 1613 "initialize_tracker_settings" 1614 ) 1615 print("Follow instructions in customization.md") 1616 pdb.set_trace() 1617 f = open(self.save_name + "/array_dims.csv", "r") 1618 for line in f: 1619 parts = line.strip().split(",") 1620 channels[parts[0]] = [int(s) for s in parts[1:]] 1621 for layer in self.neuron_module_vector: 1622 dv = layer.dendrite_module.dendrite_values[0] 1623 dv.setup_arrays(channels[layer.name]) 1624 1625 def set_optimizer_instance(self, optimizer_instance, additional_optimizers=[]): 1626 """Set optimizer instance directly. 1627 1628 Parameters 1629 ---------- 1630 optimizer_instance : object 1631 The optimizer instance to set. 1632 1633 Returns 1634 ------- 1635 None 1636 1637 """ 1638 # This call must be first before the parameters get filtered. 1639 optimizer_instance.zero_grad() 1640 try: 1641 for param_group in optimizer_instance.param_groups: 1642 if ( 1643 param_group["weight_decay"] > 0 1644 and GPA.pc.get_weight_decay_accepted() is False 1645 ): 1646 _pai_log( 1647 "warning", 1648 "For PAI training it is recommended to not use weight decay in your optimizer", 1649 ) 1650 1651 except: 1652 pass 1653 self.member_vars["optimizer_instance"] = optimizer_instance 1654 if GPA.pc.get_perforated_backpropagation(): 1655 TPB.setup_optimizer_pb(self.member_vars["optimizer_instance"]) 1656 for optimizer in additional_optimizers: 1657 TPB.filter_params(optimizer) 1658 1659 def set_optimizer(self, optimizer): 1660 """Set optimizer type to be initialized later 1661 1662 Parameters 1663 ---------- 1664 optimizer : object 1665 The optimizer type to set. 1666 1667 Returns 1668 ------- 1669 None 1670 1671 """ 1672 self.member_vars["optimizer"] = optimizer 1673 1674 def set_scheduler(self, scheduler): 1675 """Set scheduler type to be initialized later 1676 1677 Parameters 1678 ---------- 1679 scheduler : object 1680 The scheduler type to set. 1681 1682 Returns 1683 ------- 1684 None 1685 1686 """ 1687 if scheduler is not torch.optim.lr_scheduler.ReduceLROnPlateau: 1688 if GPA.pc.get_verbose(): 1689 print("Not using ReduceLROnPlateau, this is not recommended") 1690 self.member_vars["scheduler"] = scheduler 1691 1692 def increment_scheduler(self, num_ticks, mode): 1693 """Increment the scheduler a set number of times. 1694 1695 Used for finding best initial learning rate when adding dendrites. 1696 1697 Parameters 1698 ---------- 1699 num_ticks : int 1700 The number of scheduler steps to take. 1701 mode : str 1702 The mode for stepping the scheduler. Options are: 1703 - "step_learning_rate": Step based on improved accuracy epochs 1704 - "increment_epoch_count": Step based on total epoch count 1705 1706 Returns 1707 ------- 1708 current_steps : int 1709 The number of learning rate changes that occurred. 1710 learning_rate1 : float 1711 The final learning rate after stepping. 1712 1713 """ 1714 1715 current_steps = 0 1716 current_ticker = 0 1717 1718 for param_group in GPA.pai_tracker.member_vars[ 1719 "optimizer_instance" 1720 ].param_groups: 1721 learning_rate1 = param_group["lr"] 1722 1723 if GPA.pc.get_verbose(): 1724 print("Using scheduler:") 1725 print(type(self.member_vars["scheduler_instance"])) 1726 1727 while current_ticker < num_ticks: 1728 if GPA.pc.get_verbose(): 1729 print( 1730 f"Lower start rate initial {learning_rate1} " 1731 f'stepping {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} times' 1732 ) 1733 1734 if ( 1735 type(self.member_vars["scheduler_instance"]) 1736 is torch.optim.lr_scheduler.ReduceLROnPlateau 1737 ): 1738 if mode == "step_learning_rate": 1739 # Step with counter as last improved accuracy 1740 self.member_vars["scheduler_instance"].step( 1741 metrics=self.member_vars["last_improved_accuracies"][ 1742 GPA.pai_tracker.steps_after_switch() - 1 1743 ] 1744 ) 1745 elif mode == "increment_epoch_count": 1746 # Step with improved epoch counts up to current location 1747 self.member_vars["scheduler_instance"].step( 1748 metrics=self.member_vars["last_improved_accuracies"][ 1749 -((num_ticks - 1) - current_ticker) - 1 1750 ] 1751 ) 1752 else: 1753 self.member_vars["scheduler_instance"].step() 1754 1755 for param_group in GPA.pai_tracker.member_vars[ 1756 "optimizer_instance" 1757 ].param_groups: 1758 learning_rate2 = param_group["lr"] 1759 1760 if learning_rate2 != learning_rate1: 1761 current_steps += 1 1762 learning_rate1 = learning_rate2 1763 if mode == "step_learning_rate": 1764 current_ticker += 1 1765 if GPA.pc.get_verbose(): 1766 print(f"1 step {current_steps} to {learning_rate2}") 1767 1768 if mode == "increment_epoch_count": 1769 current_ticker += 1 1770 1771 return current_steps, learning_rate1 1772 1773 def setup_optimizer(self, net, opt_args, sched_args=None, parameters=None): 1774 """Initialize the optimizer and scheduler when added. 1775 1776 Parameters 1777 ---------- 1778 net : object 1779 The neural network model. 1780 opt_args : dict 1781 The arguments for the optimizer. 1782 sched_args : dict, optional 1783 The arguments for the scheduler, by default None. 1784 1785 Returns 1786 ------- 1787 optimizer : object 1788 The initialized optimizer instance. 1789 scheduler : object or None 1790 The initialized scheduler instance, or None if no scheduler was set. 1791 1792 """ 1793 if "weight_decay" in opt_args and not GPA.pc.get_weight_decay_accepted(): 1794 _pai_log( 1795 "warning", 1796 "For PAI training it is recommended to not use weight decay in your optimizer", 1797 ) 1798 1799 if ("model" not in opt_args.keys()) and "params" not in opt_args.keys(): 1800 print("In setup_optimizer it will be depreciated to not pass in params yourself in the future") 1801 print("please change the settings to include params") 1802 if self.member_vars["mode"] == "n": 1803 if parameters is not None: 1804 opt_args["params"] = parameters 1805 else: 1806 opt_args["params"] = filter(lambda p: p.requires_grad, net.parameters()) 1807 else: 1808 params = UPA.get_pai_network_params(net) 1809 if parameters is not None: 1810 # Filter parameters to only those in params, preserving weight_decay 1811 params_set = set(params) 1812 filtered_params = [] 1813 for param_group in parameters: 1814 filtered_group_params = [p for p in param_group["params"] if p in params_set] 1815 if filtered_group_params: 1816 filtered_params.append({ 1817 "params": filtered_group_params, 1818 "weight_decay": param_group["weight_decay"] 1819 }) 1820 opt_args["params"] = filtered_params 1821 else: 1822 opt_args["params"] = params 1823 elif "params" in opt_args.keys(): 1824 # Check if params is a list of param groups (dicts) or a single param group 1825 params_value = opt_args["params"] 1826 if isinstance(params_value, list) and len(params_value) > 0: 1827 # Check if it's a list of dicts (multiple param groups) or list of tensors (single group) 1828 if isinstance(params_value[0], dict): 1829 # Multiple param groups format: [{"params": [...], "lr": ...}, ...] 1830 # Filter each param group for requires_grad 1831 filtered_param_groups = [] 1832 for param_group in params_value: 1833 filtered_group_params = [p for p in param_group["params"] if p.requires_grad] 1834 if filtered_group_params: 1835 new_group = param_group.copy() 1836 new_group["params"] = filtered_group_params 1837 filtered_param_groups.append(new_group) 1838 opt_args["params"] = filtered_param_groups 1839 else: 1840 # Single param group format: [tensor1, tensor2, ...] or generator 1841 # Filter for requires_grad 1842 opt_args["params"] = [p for p in params_value if p.requires_grad] 1843 elif hasattr(params_value, '__iter__'): 1844 # Handle generators or other iterables 1845 opt_args["params"] = [p for p in params_value if p.requires_grad] 1846 1847 optimizer = self.member_vars["optimizer"](**opt_args) 1848 self.set_optimizer_instance(optimizer) 1849 1850 if self.member_vars["scheduler"] is not None: 1851 # Handle SequentialLR specially 1852 if self.member_vars["scheduler"] is torch.optim.lr_scheduler.SequentialLR: 1853 """ 1854 sched_args should be a dict with "schedulers" (list of tuples) and "milestones" 1855 For example: 1856 sequential_schedArgs = { 1857 "schedulers": [ 1858 (warmup_scheduler_class, warmup_schedArgs), 1859 (main_scheduler_class, main_schedArgs) 1860 ], 1861 "milestones": [switch_epoch] 1862 } 1863 """ 1864 schedulers = [] 1865 milestones = sched_args.get("milestones", []) 1866 scheduler_configs = sched_args.get("schedulers", []) 1867 1868 for scheduler_class, scheduler_args in scheduler_configs: 1869 schedulers.append(scheduler_class(optimizer, **scheduler_args)) 1870 1871 self.member_vars["scheduler_instance"] = torch.optim.lr_scheduler.SequentialLR( 1872 optimizer, schedulers=schedulers, milestones=milestones 1873 ) 1874 else: 1875 self.member_vars["scheduler_instance"] = self.member_vars["scheduler"]( 1876 optimizer, **sched_args 1877 ) 1878 current_steps = 0 1879 1880 for param_group in GPA.pai_tracker.member_vars[ 1881 "optimizer_instance" 1882 ].param_groups: 1883 learning_rate1 = param_group["lr"] 1884 1885 if GPA.pc.get_verbose(): 1886 print( 1887 f"Resetting scheduler with {GPA.pai_tracker.steps_after_switch()} " 1888 f"steps and {GPA.pc.get_initial_history_after_switches()} initial ticks to skip" 1889 ) 1890 1891 # Find setting of previously used learning rate before adding dendrites 1892 if ( 1893 GPA.pai_tracker.member_vars[ 1894 "current_n_learning_rate_initial_skip_steps" 1895 ] 1896 != 0 1897 ): 1898 additional_steps, learning_rate1 = self.increment_scheduler( 1899 GPA.pai_tracker.member_vars[ 1900 "current_n_learning_rate_initial_skip_steps" 1901 ], 1902 "step_learning_rate", 1903 ) 1904 current_steps += additional_steps 1905 1906 if self.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 1907 initial = GPA.pc.get_initial_history_after_switches() 1908 else: 1909 initial = 0 1910 1911 if GPA.pai_tracker.steps_after_switch() > initial: 1912 # Minus extra 1 because this gets called after start epoch 1913 additional_steps, learning_rate1 = self.increment_scheduler( 1914 (GPA.pai_tracker.steps_after_switch() - initial) - 1, 1915 "increment_epoch_count", 1916 ) 1917 current_steps += additional_steps 1918 1919 if GPA.pc.get_verbose(): 1920 print( 1921 f"Scheduler update loop with {current_steps} " 1922 f"ended with {learning_rate1}" 1923 ) 1924 print( 1925 f"Scheduler ended with {current_steps} steps " 1926 f"and lr of {learning_rate1}" 1927 ) 1928 1929 self.member_vars["current_step_count"] = current_steps 1930 return optimizer, self.member_vars["scheduler_instance"] 1931 else: 1932 return optimizer, None 1933 1934 def clear_optimizer_and_scheduler(self): 1935 """Clear the instances for saving. 1936 1937 Parameters 1938 ---------- 1939 None 1940 1941 Returns 1942 ------- 1943 None 1944 This function does not return a value. 1945 """ 1946 self.member_vars["optimizer_instance"] = None 1947 self.member_vars["scheduler_instance"] = None 1948 1949 def switch_time(self): 1950 """Determine if it's time to switch between neuron and dendrite training. 1951 1952 Parameters 1953 ---------- 1954 None 1955 1956 Returns 1957 ------- 1958 bool 1959 True if it's time to switch, False otherwise. 1960 1961 Notes 1962 ----- 1963 Based on current settings and history of scores. 1964 """ 1965 1966 switch_phrase = "No mode, this should never be the case." 1967 switch_number = GPA.pc.get_n_epochs_to_switch() 1968 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1969 switch_phrase = "DOING_SWITCH_EVERY_TIME" 1970 elif self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY: 1971 switch_phrase = "DOING_HISTORY" 1972 elif self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH: 1973 switch_phrase = "DOING_FIXED_SWITCH" 1974 switch_number = GPA.pc.get_fixed_switch_num() 1975 elif self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1976 switch_phrase = "DOING_NO_SWITCH" 1977 else: 1978 print( 1979 "A switch mode must be set. Check your settings for GPA.pc.set_switch_mode()." 1980 ) 1981 pdb.set_trace() 1982 if not GPA.pc.get_silent(): 1983 if(GPA.pc.get_perforated_backpropagation()): 1984 print( 1985 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1986 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1987 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1988 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1989 f'n: {switch_number}, p: {GPA.pc.get_p_epochs_to_switch()}, ' 1990 f'num_cycles: {self.member_vars["num_cycles"]}' 1991 ) 1992 else: 1993 print( 1994 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1995 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1996 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1997 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1998 f'n: {switch_number}, num_cycles: {self.member_vars["num_cycles"]}' 1999 ) 2000 print( 2001 f' Score tracking: current_n_set_global_best={self.member_vars["current_n_set_global_best"]}, ' 2002 f'global_best={self.member_vars["global_best_validation_score"]:.4f}, ' 2003 f'current_best={self.member_vars["current_best_validation_score"]:.4f}' 2004 ) 2005 if GPA.pc.get_perforated_backpropagation(): 2006 # this will fill in epoch last improved 2007 TPB.best_pai_score_improved_this_epoch(self) ## CLOSED ONLY 2008 if self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 2009 if not GPA.pc.get_silent(): 2010 print("Returning False - doing no switch mode") 2011 return False 2012 2013 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 2014 if not GPA.pc.get_silent(): 2015 print("Returning True - switching every time") 2016 return True 2017 2018 # Check if we're in the middle of learning rate optimization 2019 # If so, block ALL switch triggers until committed 2020 if GPA.pc.get_verbose(): 2021 print("=== LR Optimization Check ===") 2022 print(f' mode == "n": {self.member_vars["mode"] == "n"}') 2023 print(f" get_learn_dendrites_live(): {GPA.pc.get_learn_dendrites_live()}") 2024 print(f' committed_to_initial_rate: {GPA.pai_tracker.member_vars["committed_to_initial_rate"]}') 2025 print(f" get_dont_give_up_unless_learning_rate_lowered(): {GPA.pc.get_dont_give_up_unless_learning_rate_lowered()}") 2026 print(f' current_n_learning_rate_initial_skip_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"]}') 2027 print(f' last_max_learning_rate_steps: {self.member_vars["last_max_learning_rate_steps"]}') 2028 print(f' skip_steps < max_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"] < self.member_vars["last_max_learning_rate_steps"]}') 2029 print(f' scheduler is not None: {self.member_vars["scheduler"] is not None}') 2030 print("=============================") 2031 2032 if ( 2033 ((self.member_vars["mode"] == "n") or GPA.pc.get_learn_dendrites_live()) 2034 and (GPA.pai_tracker.member_vars["committed_to_initial_rate"] is False) 2035 and (GPA.pc.get_dont_give_up_unless_learning_rate_lowered()) 2036 and ( 2037 self.member_vars["current_n_learning_rate_initial_skip_steps"] 2038 <= self.member_vars["last_max_learning_rate_steps"] 2039 ) 2040 and self.member_vars["scheduler"] is not None 2041 ): 2042 if not GPA.pc.get_silent(): 2043 print( 2044 f"Returning False - learning rate optimization in progress. " 2045 f"Not committed yet. Comparing " 2046 f'initial {self.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 2047 f'to last max {self.member_vars["last_max_learning_rate_steps"]}' 2048 ) 2049 return False 2050 2051 if len(self.member_vars["switch_epochs"]) == 0: 2052 this_count = self.member_vars["num_epochs_run"] 2053 else: 2054 this_count = ( 2055 self.member_vars["num_epochs_run"] 2056 - self.member_vars["switch_epochs"][-1] 2057 ) 2058 cap_switch = False 2059 if GPA.pc.get_perforated_backpropagation(): 2060 cap_switch = TPB.check_cap_switch(self, this_count) 2061 2062 if self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY and ( 2063 ( 2064 (self.member_vars["mode"] == "n") 2065 and ( 2066 self.member_vars["num_epochs_run"] 2067 - self.member_vars["epoch_last_improved"] 2068 >= GPA.pc.get_n_epochs_to_switch() 2069 ) 2070 and this_count 2071 >= GPA.pc.get_initial_history_after_switches() 2072 + GPA.pc.get_n_epochs_to_switch() 2073 ) 2074 or (GPA.pc.get_perforated_backpropagation() and TPB.history_switch(self)) 2075 or cap_switch 2076 ): 2077 if not GPA.pc.get_silent(): 2078 print("Returning True - History and last improved is hit") 2079 return True 2080 2081 if self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH and ( 2082 ( 2083 self.member_vars["total_epochs_run"] % GPA.pc.get_fixed_switch_num() 2084 == GPA.pc.get_fixed_switch_num() - 1 2085 ) 2086 and self.member_vars["num_epochs_run"] 2087 >= GPA.pc.get_first_fixed_switch_num() - 1 2088 ): 2089 if not GPA.pc.get_silent(): 2090 print("Returning True - Fixed switch number is hit") 2091 return True 2092 2093 if not GPA.pc.get_silent(): 2094 print("Returning False - no triggers to switch have been hit") 2095 return False 2096 2097 def steps_after_switch(self): 2098 """Based on settings, return value for steps since a switch. 2099 2100 Different options for param vals setting determine what is returned. 2101 2102 Parameters 2103 ---------- 2104 None 2105 2106 Returns 2107 ------- 2108 int 2109 The number of epochs since the last switch, or total epochs run, 2110 depending on settings. 2111 2112 """ 2113 if self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_TOTAL_EPOCH: 2114 return self.member_vars["num_epochs_run"] 2115 elif ( 2116 self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_UPDATE_EPOCH 2117 ): 2118 return self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2119 elif ( 2120 self.member_vars["param_vals_setting"] 2121 == GPA.pc.PARAM_VALS_BY_NEURON_EPOCH_START 2122 ): 2123 if self.member_vars["mode"] == "p": 2124 return ( 2125 self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2126 ) 2127 else: 2128 return self.member_vars["num_epochs_run"] 2129 else: 2130 print( 2131 f'{self.member_vars["param_vals_setting"]} is not a valid param vals option' 2132 ) 2133 pdb.set_trace() 2134 2135 def add_pai_neuron_module(self, new_module, initial_add=True): 2136 """Add neuron modules to internal vectors. 2137 2138 Parameters 2139 ---------- 2140 new_module : object 2141 The new module to add. 2142 initial_add : bool, optional 2143 Whether this is the initial addition rather than loading from file 2144 2145 Returns 2146 ------- 2147 None 2148 2149 """ 2150 2151 # If it's a duplicate, ignore the second addition 2152 if new_module in self.neuron_module_vector: 2153 return 2154 self.neuron_module_vector.append(new_module) 2155 if self.member_vars["doing_pai"]: 2156 PA.set_wrapped_params(new_module) 2157 if initial_add: 2158 self.member_vars["best_scores"].append([]) 2159 self.member_vars["current_scores"].append([]) 2160 2161 def add_tracked_neuron_module(self, new_module, initial_add=True): 2162 """Add tracked modules to internal vectors 2163 2164 Parameters 2165 ---------- 2166 new_module : object 2167 The new module to add. 2168 initial_add : bool, optional 2169 Whether this is the initial addition rather than loading from file 2170 2171 Returns 2172 ------- 2173 None 2174 2175 """ 2176 # If it's a duplicate, ignore the second addition 2177 if new_module in self.tracked_neuron_module_vector: 2178 return 2179 self.tracked_neuron_module_vector.append(new_module) 2180 if self.member_vars["doing_pai"]: 2181 PA.set_tracked_params(new_module) 2182 2183 def reset_module_vector(self, net, load_from_restart): 2184 """Clear internal vectors and reset from network. 2185 2186 Parameters 2187 ---------- 2188 net : object 2189 The neural network model. 2190 load_from_restart : bool 2191 Whether loading from a restart file. 2192 2193 Returns 2194 ------- 2195 None 2196 2197 """ 2198 self.neuron_module_vector = [] 2199 self.tracked_neuron_module_vector = [] 2200 this_list = UPA.get_pai_modules(net, 0) 2201 for module in this_list: 2202 self.add_pai_neuron_module(module, initial_add=load_from_restart) 2203 this_list = UPA.get_tracked_modules(net, 0) 2204 for module in this_list: 2205 self.add_tracked_neuron_module(module, initial_add=load_from_restart) 2206 2207 def reset_vals_for_score_reset(self): 2208 """Reset cycle scores for new cycle. 2209 2210 Parameters 2211 ---------- 2212 None 2213 2214 Returns 2215 ------- 2216 None 2217 This function does not return a value. 2218 """ 2219 2220 if GPA.pc.get_find_best_lr(): 2221 self.member_vars["committed_to_initial_rate"] = False 2222 print("Resetting committed to initial rate to False") 2223 # If retaining all dendrties always say that the current dendrites set global best for saving and loading 2224 if GPA.pc.get_retain_all_dendrites(): 2225 self.member_vars["current_n_set_global_best"] = True 2226 self.member_vars["global_best_validation_score"] = 0 2227 else: 2228 self.member_vars["current_n_set_global_best"] = False 2229 2230 # Don't reset global best, but do reset current best 2231 self.member_vars["current_best_validation_score"] = 0 2232 self.member_vars["initial_lr_test_epoch_count"] = -1 2233 2234 def set_dendrite_training(self): 2235 """Signal all layers to start dendrite training. 2236 2237 Parameters 2238 ---------- 2239 None 2240 2241 Returns 2242 ------- 2243 None 2244 This function does not return a value. 2245 """ 2246 if GPA.pc.get_verbose(): 2247 print("Calling set_dendrite_training") 2248 2249 for layer in self.neuron_module_vector[:]: 2250 worked = layer.set_mode("p") 2251 """ 2252 worked is False when a layer was added to the neuron module vector 2253 but then it's never actually been used. This can happen when 2254 you have set a layer to have requires_grad = False or when 2255 you have a module as a member variable but it's not actually 2256 part of the network. Should be moved to be a tracked layer 2257 rather than a neuron layer. 2258 """ 2259 if not worked: 2260 self.neuron_module_vector.remove(layer) 2261 2262 for layer in self.tracked_neuron_module_vector[:]: 2263 worked = layer.set_mode("p") 2264 2265 self.create_new_dendrite_module() 2266 self.member_vars["mode"] = "p" 2267 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2268 2269 if GPA.pc.get_learn_dendrites_live(): 2270 self.reset_vals_for_score_reset() 2271 2272 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2273 "current_step_count" 2274 ] 2275 2276 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 2277 GPA.pai_tracker.member_vars["num_cycles"] += 1 2278 2279 2280 def set_neuron_training(self): 2281 """Signal all layers to start neuron training. 2282 2283 Parameters 2284 ---------- 2285 None 2286 2287 Returns 2288 ------- 2289 None 2290 This function does not return a value. 2291 """ 2292 for module in self.neuron_module_vector: 2293 module.set_mode("n") 2294 for module in self.tracked_neuron_module_vector[:]: 2295 module.set_mode("n") 2296 2297 self.member_vars["mode"] = "n" 2298 self.member_vars["num_dendrites_added"] += 1 2299 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2300 self.reset_vals_for_score_reset() 2301 2302 self.member_vars["current_cycle_lr_max_scores"] = [] 2303 if GPA.pc.get_learn_dendrites_live(): 2304 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2305 "current_step_count" 2306 ] 2307 GPA.pai_tracker.member_vars["num_cycles"] += 1 2308 2309 if GPA.pc.get_reset_best_score_on_switch(): 2310 GPA.pai_tracker.member_vars["current_best_validation_score"] = 0 2311 GPA.pai_tracker.member_vars["running_accuracy"] = 0 2312 2313 def start_epoch(self, internal_call=False): 2314 """Perform steps for when a new training epoch is about to begin. 2315 2316 Parameters 2317 ---------- 2318 internal_call : bool, optional 2319 Whether this is an internal call or manual call 2320 2321 Returns 2322 ------- 2323 None 2324 2325 Notes 2326 ----- 2327 If you ever need to call this manually, set internal_call to False. 2328 2329 """ 2330 if self.member_vars["manual_train_switch"] and internal_call: 2331 return 2332 2333 if not internal_call and not self.member_vars["manual_train_switch"]: 2334 self.member_vars["manual_train_switch"] = True 2335 self.saved_time = 0 2336 self.member_vars["num_epochs_run"] = -1 2337 self.member_vars["total_epochs_run"] = -1 2338 2339 end = time.time() 2340 if self.member_vars["manual_train_switch"]: 2341 if self.saved_time != 0: 2342 if self.member_vars["mode"] == "p": 2343 self.member_vars["p_val_times"].append(end - self.saved_time) 2344 else: 2345 self.member_vars["n_val_times"].append(end - self.saved_time) 2346 2347 if self.member_vars["mode"] == "p": 2348 for layer in self.neuron_module_vector: 2349 for m in range(0, GPA.pc.get_global_candidates()): 2350 with torch.no_grad(): 2351 if GPA.pc.get_verbose(): 2352 print(f"Resetting score for {layer.name}") 2353 # Snapshot best_score before reset so we can compute per-epoch improvement 2354 layer.dendrite_module.dendrite_values[ 2355 m 2356 ].epoch_start_best_score.copy_( 2357 layer.dendrite_module.dendrite_values[ 2358 m 2359 ].best_score.detach() 2360 ) 2361 layer.dendrite_module.dendrite_values[ 2362 m 2363 ].best_score_improved_this_epoch = ( 2364 layer.dendrite_module.dendrite_values[ 2365 m 2366 ].best_score_improved_this_epoch 2367 * 0 2368 ) 2369 layer.dendrite_module.dendrite_values[ 2370 m 2371 ].nodes_best_improved_this_epoch = ( 2372 layer.dendrite_module.dendrite_values[ 2373 m 2374 ].nodes_best_improved_this_epoch 2375 * 0 2376 ) 2377 layer.dendrite_module.dendrite_values[ 2378 m 2379 ].nodes_improved_any = ( 2380 layer.dendrite_module.dendrite_values[ 2381 m 2382 ].nodes_improved_any 2383 * 0 2384 ) 2385 if GPA.pc.get_perforated_backpropagation(): 2386 self.member_vars["best_mean_score_improved_this_epoch"] = 0 2387 self.member_vars["num_epochs_run"] += 1 2388 self.member_vars["total_epochs_run"] = ( 2389 self.member_vars["num_epochs_run"] + self.member_vars["overwritten_epochs"] 2390 ) 2391 self.saved_time = end 2392 2393 def stop_epoch(self, internal_call=False): 2394 """Perform steps when a training epoch has completed. 2395 2396 Parameters 2397 ---------- 2398 internal_call : bool, optional 2399 Whether this is an internal call or manual call 2400 2401 Returns 2402 ------- 2403 None 2404 2405 Notes 2406 ----- 2407 If you ever need to call this manually, set internal_call to False. 2408 2409 """ 2410 end = time.time() 2411 if self.member_vars["manual_train_switch"] and internal_call: 2412 return 2413 2414 if self.member_vars["manual_train_switch"]: 2415 if self.member_vars["mode"] == "p": 2416 self.member_vars["p_train_times"].append(end - self.saved_time) 2417 else: 2418 self.member_vars["n_train_times"].append(end - self.saved_time) 2419 else: 2420 if self.member_vars["mode"] == "p": 2421 self.member_vars["p_epoch_times"].append(end - self.saved_time) 2422 else: 2423 self.member_vars["n_epoch_times"].append(end - self.saved_time) 2424 2425 self.saved_time = end 2426 2427 def initialize( 2428 self, 2429 model, 2430 doing_pai=True, 2431 save_name="PAI", 2432 making_graphs=True, 2433 maximizing_score=True, 2434 num_classes=10000, 2435 values_per_train_epoch=-1, 2436 values_per_val_epoch=-1, 2437 zooming_graph=True, 2438 ): 2439 """Setup the tracker with initial settings. 2440 2441 2442 Parameters 2443 ---------- 2444 model : object 2445 The neural network model. 2446 doing_pai : bool, optional 2447 Whether to add dendrites, by default True. 2448 save_name : str, optional 2449 The name under which to save the model. 2450 making_graphs : bool, optional 2451 Whether to make graphs, by default True. 2452 maximizing_score : bool, optional 2453 Whether to maximize the score, by default True. 2454 num_classes : int, optional 2455 The number of classes in the dataset, unused 2456 values_per_train_epoch : int, optional 2457 The number of values to look back for graphing 2458 during training, by default -1 (all values). 2459 values_per_val_epoch : int, optional 2460 The number of values to look back for graphing 2461 during validation, by default -1 (all values). 2462 zooming_graph : bool, optional 2463 Whether to zoom on graphs, by default True. 2464 2465 2466 Returns 2467 ------- 2468 nn.Module 2469 Converted model instance configured for the tracker settings. 2470 """ 2471 model = UPA.convert_network(model) 2472 self.member_vars["doing_pai"] = doing_pai 2473 self.member_vars["maximizing_score"] = maximizing_score 2474 self.save_name = save_name 2475 self.zooming_graph = zooming_graph 2476 self.making_graphs = making_graphs 2477 2478 if not self.loaded: 2479 self.member_vars["running_accuracy"] = (1.0 / num_classes) * 100 2480 2481 self.values_per_train_epoch = values_per_train_epoch 2482 self.values_per_val_epoch = values_per_val_epoch 2483 2484 if GPA.pc.get_testing_dendrite_capacity(): 2485 if not GPA.pc.get_silent(): 2486 print("Running a test of Dendrite Capacity.") 2487 GPA.pc.set_switch_mode(GPA.pc.DOING_SWITCH_EVERY_TIME) 2488 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 2489 GPA.pc.set_retain_all_dendrites(True) 2490 GPA.pc.set_max_dendrite_tries(1000) 2491 GPA.pc.set_max_dendrites(1000) 2492 if GPA.pc.get_perforated_backpropagation(): 2493 GPA.pc.set_initial_correlation_batches(1) 2494 else: 2495 if not GPA.pc.get_silent(): 2496 print("Running Dendrite Experiment") 2497 return model 2498 2499 def generate_accuracy_plots(self, ax, save_folder, extra_string): 2500 """ 2501 Generate plots and csvs for accuracy 2502 2503 Parameters 2504 ---------- 2505 ax : object 2506 The matplotlib axis to plot on. 2507 save_folder : str 2508 The folder to save the plots and csvs in. 2509 extra_string : str 2510 An extra string to append to the filenames. 2511 2512 Returns 2513 ------- 2514 None 2515 2516 """ 2517 2518 # If scores are being saved for epochs that get overwritten, plot them 2519 for list_id in range(len(self.member_vars["overwritten_extras"])): 2520 for extra_id in self.member_vars["overwritten_extras"][list_id]: 2521 ax.plot( 2522 np.arange( 2523 len(self.member_vars["overwritten_extras"][list_id][extra_id]) 2524 ), 2525 self.member_vars["overwritten_extras"][list_id][extra_id], 2526 color=_PAI_PALETTE["train"], 2527 alpha=0.35, 2528 ) 2529 ax.plot( 2530 np.arange(len(self.member_vars["overwritten_vals"][list_id])), 2531 self.member_vars["overwritten_vals"][list_id], 2532 color=_PAI_PALETTE["validation"], 2533 alpha=0.35, 2534 ) 2535 2536 # Determine which accuracy vector to use 2537 if GPA.pc.get_drawing_pai(): 2538 accuracies = self.member_vars["accuracies"] 2539 else: 2540 accuracies = self.member_vars["n_accuracies"] 2541 2542 # Get pointer to additional scores being saved 2543 extra_scores = self.member_vars["extra_scores"] 2544 2545 # Plot the main accuracy scores 2546 ax.plot( 2547 np.arange(len(accuracies)), 2548 accuracies, 2549 label="Validation Scores", 2550 color=_PAI_PALETTE["validation"], 2551 ) 2552 ax.plot( 2553 np.arange(len(self.member_vars["running_accuracies"])), 2554 self.member_vars["running_accuracies"], 2555 label="Validation Running Scores", 2556 color=_PAI_PALETTE["validation"], 2557 alpha=0.4, 2558 ) 2559 2560 # Plot additional scores 2561 for extra_index, extra_score in enumerate(extra_scores): 2562 ax.plot( 2563 np.arange(len(extra_scores[extra_score])), 2564 extra_scores[extra_score], 2565 label=extra_score, 2566 color=_pai_extra_score_color(extra_score, extra_index), 2567 ) 2568 2569 plt.title(save_folder + "/" + self.save_name + "Scores") 2570 plt.xlabel("Epochs") 2571 plt.ylabel("Score") 2572 2573 # Add point at epoch last improved and best validation score 2574 if GPA.pc.get_drawing_pai(): 2575 ax.plot( 2576 self.member_vars["epoch_last_improved"], 2577 self.member_vars["global_best_validation_score"], 2578 "o", 2579 color=_PAI_PALETTE["marker_best"], 2580 label="Global best (y)", 2581 ) 2582 ax.plot( 2583 self.member_vars["epoch_last_improved"], 2584 accuracies[self.member_vars["epoch_last_improved"]], 2585 "o", 2586 color=_PAI_PALETTE["marker_improved"], 2587 label="Epoch Last Improved", 2588 ) 2589 else: 2590 if self.member_vars["mode"] == "n": 2591 missed_time = ( 2592 self.member_vars["num_epochs_run"] 2593 - self.member_vars["epoch_last_improved"] 2594 ) 2595 ax.plot( 2596 (len(self.member_vars["n_accuracies"]) - 1) - missed_time, 2597 self.member_vars["n_accuracies"][-(missed_time + 1)], 2598 "o", 2599 color=_PAI_PALETTE["marker_improved"], 2600 label="Epoch Last Improved", 2601 ) 2602 2603 # Generate csv file for the values graphed 2604 pd1 = pd.DataFrame( 2605 {"Epochs": np.arange(len(accuracies)), "Validation Scores": accuracies} 2606 ) 2607 pd2 = pd.DataFrame( 2608 { 2609 "Epochs": np.arange(len(self.member_vars["running_accuracies"])), 2610 "Validation Running Scores": self.member_vars["running_accuracies"], 2611 } 2612 ) 2613 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2614 for extra_score in extra_scores: 2615 pd2 = pd.DataFrame( 2616 { 2617 "Epochs": np.arange(len(extra_scores[extra_score])), 2618 extra_score: extra_scores[extra_score], 2619 } 2620 ) 2621 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2622 extra_scores_without_graphing = self.member_vars[ 2623 "extra_scores_without_graphing" 2624 ] 2625 for extra_score in extra_scores_without_graphing: 2626 pd2 = pd.DataFrame( 2627 { 2628 "Epochs": np.arange( 2629 len(extra_scores_without_graphing[extra_score]) 2630 ), 2631 extra_score: extra_scores_without_graphing[extra_score], 2632 } 2633 ) 2634 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2635 pd1.to_csv( 2636 save_folder + "/" + self.save_name + extra_string + "Scores.csv", 2637 index=False, 2638 ) 2639 del pd1, pd2 2640 2641 # Set y min and max to zoom in on important part of axis 2642 if ( 2643 len(self.member_vars["switch_epochs"]) > 0 2644 and self.member_vars["switch_epochs"][0] > 0 2645 and self.zooming_graph 2646 ): 2647 if GPA.pai_tracker.member_vars["maximizing_score"]: 2648 min_val = np.array( 2649 accuracies[0 : self.member_vars["switch_epochs"][0]] 2650 ).mean() 2651 for extra_score in extra_scores: 2652 min_pot = np.array( 2653 extra_scores[extra_score][ 2654 0 : self.member_vars["switch_epochs"][0] 2655 ] 2656 ).mean() 2657 if min_pot < min_val: 2658 min_val = min_pot 2659 ax.set_ylim(ymin=min_val) 2660 else: 2661 max_val = np.array( 2662 accuracies[0 : self.member_vars["switch_epochs"][0]] 2663 ).mean() 2664 for extra_score in extra_scores: 2665 max_pot = np.array( 2666 extra_scores[extra_score][ 2667 0 : self.member_vars["switch_epochs"][0] 2668 ] 2669 ).mean() 2670 if max_pot > max_val: 2671 max_val = max_pot 2672 ax.set_ylim(ymax=max_val) 2673 2674 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2675 _pai_grid(ax) 2676 2677 # Draw vertical lines for epochs where a dendrite switch occurred. 2678 # Colour by the phase being entered: dendrite/PAI phase -> teal, 2679 # normal phase -> orange, alternating starting with dendrite. 2680 if GPA.pc.get_drawing_pai() and self.member_vars["doing_pai"]: 2681 for i, switcher in enumerate(self.member_vars["switch_epochs"]): 2682 # Mode always starts at "n" and switch_epochs alternates 2683 # n->p->n->p..., so the first switch enters the dendrite 2684 # phase and every other one alternates from there. 2685 entering_p = (i % 2) == 0 2686 plt.axvline( 2687 x=switcher, 2688 ymin=0, 2689 ymax=1, 2690 color=_PAI_PALETTE["dendrite"] 2691 if entering_p 2692 else _PAI_PALETTE["normal"], 2693 ) 2694 else: 2695 for switcher in self.member_vars["n_switch_epochs"]: 2696 plt.axvline( 2697 x=switcher, ymin=0, ymax=1, color=_PAI_PALETTE["normal"] 2698 ) 2699 2700 def generate_time_plots(self, ax, save_folder, extra_string): 2701 """ 2702 Generate plots and csvs for timing 2703 2704 Parameters 2705 ---------- 2706 ax : object 2707 The matplotlib axis to plot on. 2708 save_folder : str 2709 The folder to save the plots and csvs in. 2710 extra_string : str 2711 An extra string to append to the filenames. 2712 2713 Returns 2714 ------- 2715 None 2716 2717 """ 2718 if self.member_vars["manual_train_switch"]: 2719 ax.plot( 2720 np.arange(len(self.member_vars["n_train_times"])), 2721 self.member_vars["n_train_times"], 2722 label="Normal Epoch Train Times", 2723 color=_PAI_PALETTE["normal"], 2724 ) 2725 ax.plot( 2726 np.arange(len(self.member_vars["p_train_times"])), 2727 self.member_vars["p_train_times"], 2728 label="PAI Epoch Train Times", 2729 color=_PAI_PALETTE["dendrite"], 2730 ) 2731 ax.plot( 2732 np.arange(len(self.member_vars["n_val_times"])), 2733 self.member_vars["n_val_times"], 2734 label="Normal Epoch Val Times", 2735 color=_PAI_PALETTE["normal"], 2736 alpha=0.45, 2737 ) 2738 ax.plot( 2739 np.arange(len(self.member_vars["p_val_times"])), 2740 self.member_vars["p_val_times"], 2741 label="PAI Epoch Val Times", 2742 color=_PAI_PALETTE["dendrite"], 2743 alpha=0.45, 2744 ) 2745 2746 plt.title( 2747 save_folder + "/" + self.save_name + "times (by train() and eval())" 2748 ) 2749 plt.xlabel("Iteration") 2750 plt.ylabel("Epoch Time in Seconds ") 2751 ax.set_ylim(ymin=0) 2752 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2753 _pai_grid(ax) 2754 2755 pd1 = pd.DataFrame( 2756 { 2757 "Epochs": np.arange(len(self.member_vars["n_train_times"])), 2758 "Normal Epoch Train Times": self.member_vars["n_train_times"], 2759 } 2760 ) 2761 pd2 = pd.DataFrame( 2762 { 2763 "Epochs": np.arange(len(self.member_vars["p_train_times"])), 2764 "PAI Epoch Train Times": self.member_vars["p_train_times"], 2765 } 2766 ) 2767 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2768 2769 pd2 = pd.DataFrame( 2770 { 2771 "Epochs": np.arange(len(self.member_vars["n_val_times"])), 2772 "Normal Epoch Val Times": self.member_vars["n_val_times"], 2773 } 2774 ) 2775 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2776 2777 pd2 = pd.DataFrame( 2778 { 2779 "Epochs": np.arange(len(self.member_vars["p_val_times"])), 2780 "PAI Epoch Val Times": self.member_vars["p_val_times"], 2781 } 2782 ) 2783 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2784 2785 pd1.to_csv( 2786 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2787 index=False, 2788 ) 2789 del pd1, pd2 2790 else: 2791 ax.plot( 2792 np.arange(len(self.member_vars["n_epoch_times"])), 2793 self.member_vars["n_epoch_times"], 2794 label="Normal Epoch Times", 2795 color=_PAI_PALETTE["normal"], 2796 ) 2797 ax.plot( 2798 np.arange(len(self.member_vars["p_epoch_times"])), 2799 self.member_vars["p_epoch_times"], 2800 label="PAI Epoch Times", 2801 color=_PAI_PALETTE["dendrite"], 2802 ) 2803 2804 plt.title( 2805 save_folder + "/" + self.save_name + "times (by train() and eval())" 2806 ) 2807 plt.xlabel("Iteration") 2808 plt.ylabel("Epoch Time in Seconds ") 2809 ax.set_ylim(ymin=0) 2810 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2811 _pai_grid(ax) 2812 2813 pd1 = pd.DataFrame( 2814 { 2815 "Epochs": np.arange(len(self.member_vars["n_epoch_times"])), 2816 "Normal Epoch Times": self.member_vars["n_epoch_times"], 2817 } 2818 ) 2819 pd2 = pd.DataFrame( 2820 { 2821 "Epochs": np.arange(len(self.member_vars["p_epoch_times"])), 2822 "PAI Epoch Times": self.member_vars["p_epoch_times"], 2823 } 2824 ) 2825 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2826 2827 pd1.to_csv( 2828 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2829 index=False, 2830 ) 2831 del pd1, pd2 2832 2833 if self.values_per_train_epoch != -1 and self.values_per_val_epoch != -1: 2834 ax2 = ax.twinx() # Second axes sharing same x-axis 2835 ax2.set_ylabel("Single Datapoint Time in Seconds") 2836 2837 ax2.plot( 2838 np.arange(len(self.member_vars["n_train_times"])), 2839 np.array(self.member_vars["n_train_times"]) 2840 / self.values_per_train_epoch, 2841 linestyle="dashed", 2842 label="Normal Train Item Times", 2843 color=_PAI_PALETTE["normal"], 2844 ) 2845 ax2.plot( 2846 np.arange(len(self.member_vars["p_train_times"])), 2847 np.array(self.member_vars["p_train_times"]) 2848 / self.values_per_train_epoch, 2849 linestyle="dashed", 2850 label="PAI Train Item Times", 2851 color=_PAI_PALETTE["dendrite"], 2852 ) 2853 ax2.plot( 2854 np.arange(len(self.member_vars["n_val_times"])), 2855 np.array(self.member_vars["n_val_times"]) / self.values_per_val_epoch, 2856 linestyle="dashed", 2857 label="Normal Val Item Times", 2858 color=_PAI_PALETTE["normal"], 2859 alpha=0.45, 2860 ) 2861 ax2.plot( 2862 np.arange(len(self.member_vars["p_val_times"])), 2863 np.array(self.member_vars["p_val_times"]) / self.values_per_val_epoch, 2864 linestyle="dashed", 2865 label="PAI Val Item Times", 2866 color=_PAI_PALETTE["dendrite"], 2867 alpha=0.45, 2868 ) 2869 ax2.tick_params(axis="y") 2870 ax2.set_ylim(ymin=0) 2871 ax2.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2872 2873 def generate_learning_rate_plots(self, ax, save_folder, extra_string): 2874 """ 2875 Generate plots and csvs for learning rate 2876 2877 Parameters 2878 ---------- 2879 ax : object 2880 The matplotlib axis to plot on. 2881 save_folder : str 2882 The folder to save the plots and csvs in. 2883 extra_string : str 2884 An extra string to append to the filenames. 2885 2886 Returns 2887 ------- 2888 None 2889 2890 """ 2891 ax.plot( 2892 np.arange(len(self.member_vars["training_learning_rates"])), 2893 self.member_vars["training_learning_rates"], 2894 label="learning_rate", 2895 color=_PAI_PALETTE["learning_rate"], 2896 ) 2897 plt.title(save_folder + "/" + self.save_name + "learning_rate") 2898 plt.xlabel("Epochs") 2899 plt.ylabel("learning_rate") 2900 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2901 _pai_grid(ax) 2902 2903 pd1 = pd.DataFrame( 2904 { 2905 "Epochs": np.arange(len(self.member_vars["training_learning_rates"])), 2906 "learning_rate": self.member_vars["training_learning_rates"], 2907 } 2908 ) 2909 pd1.to_csv( 2910 save_folder + "/" + self.save_name + extra_string + "learning_rate.csv", 2911 index=False, 2912 ) 2913 del pd1 2914 2915 def get_current_pb_scores(self): 2916 """ 2917 Get the latest best PBScore of each dendrite layer, the same numbers 2918 written to the Best PBScores csv. 2919 2920 Returns 2921 ------- 2922 dict 2923 Layer name to score. Empty outside of dendrite scoring phases, 2924 when no candidate dendrites are being scored. 2925 2926 """ 2927 if not self.member_vars["doing_pai"]: 2928 return {} 2929 if not GPA.pc.get_perforated_backpropagation(): 2930 return {} 2931 # Scores only advance while candidate dendrites are being trained 2932 if ( 2933 self.member_vars["mode"] != "p" 2934 and not GPA.pc.get_learn_dendrites_live() 2935 ): 2936 return {} 2937 2938 scores = {} 2939 for layer_id in range(len(self.neuron_module_vector)): 2940 if layer_id >= len(self.member_vars["best_scores"]): 2941 continue 2942 layer_scores = self.member_vars["best_scores"][layer_id] 2943 if len(layer_scores) == 0: 2944 continue 2945 score = layer_scores[-1] 2946 if hasattr(score, "item"): 2947 score = score.item() 2948 score = float(score) 2949 if math.isnan(score) or math.isinf(score): 2950 continue 2951 scores[self.neuron_module_vector[layer_id].name] = score 2952 return scores 2953 2954 def get_current_pb_scores_current(self): 2955 """ 2956 Like :meth:`get_current_pb_scores` but returns the *current* (not best) 2957 PBScore of each dendrite layer - ``current_scores[layer][-1]`` - the 2958 dashed series in ``generate_dendrite_learning_plots``. 2959 2960 Returns 2961 ------- 2962 dict 2963 Layer name to score. Empty outside of dendrite scoring phases. 2964 """ 2965 if not self.member_vars["doing_pai"]: 2966 return {} 2967 if not GPA.pc.get_perforated_backpropagation(): 2968 return {} 2969 if ( 2970 self.member_vars["mode"] != "p" 2971 and not GPA.pc.get_learn_dendrites_live() 2972 ): 2973 return {} 2974 2975 scores = {} 2976 for layer_id in range(len(self.neuron_module_vector)): 2977 if layer_id >= len(self.member_vars["current_scores"]): 2978 continue 2979 layer_scores = self.member_vars["current_scores"][layer_id] 2980 if len(layer_scores) == 0: 2981 continue 2982 score = layer_scores[-1] 2983 if hasattr(score, "item"): 2984 score = score.item() 2985 score = float(score) 2986 if math.isnan(score) or math.isinf(score): 2987 continue 2988 scores[self.neuron_module_vector[layer_id].name] = score 2989 return scores 2990 2991 def generate_dendrite_learning_plots(self, ax, save_folder, extra_string): 2992 """ 2993 Generate dendrite score plots for the tracker. 2994 Also saves csv files associated with the plots. 2995 2996 Parameters 2997 ---------- 2998 ax : matplotlib.axes.Axes 2999 Axis used for plotting dendrite-learning curves. 3000 save_folder : str 3001 Directory where plot images and CSV summaries are written. 3002 extra_string : str 3003 Filename suffix used to distinguish this output set. 3004 3005 Returns 3006 ------- 3007 None 3008 Saves plots and score CSV files to disk. 3009 """ 3010 if self.member_vars["doing_pai"]: 3011 pd1 = None 3012 pd2 = None 3013 num_colors = len(self.neuron_module_vector) 3014 3015 layer_colors = [ 3016 _PAI_DENDRITE_CYCLE[i % len(_PAI_DENDRITE_CYCLE)] 3017 for i in range(num_colors) 3018 ] 3019 3020 for layer_id in range(len(self.neuron_module_vector)): 3021 color = layer_colors[layer_id] 3022 ax.plot( 3023 np.arange(len(self.member_vars["best_scores"][layer_id])), 3024 self.member_vars["best_scores"][layer_id], 3025 label=self.neuron_module_vector[layer_id].name, 3026 color=color, 3027 ) 3028 3029 pd2 = pd.DataFrame( 3030 { 3031 "Epochs": np.arange( 3032 len(self.member_vars["best_scores"][layer_id]) 3033 ), 3034 f"Best ever for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 3035 "best_scores" 3036 ][ 3037 layer_id 3038 ], 3039 } 3040 ) 3041 3042 if pd1 is None: 3043 pd1 = pd2 3044 else: 3045 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 3046 3047 if len(self.member_vars["current_scores"][layer_id]) != 0: 3048 ax.plot( 3049 np.arange(len(self.member_vars["current_scores"][layer_id])), 3050 self.member_vars["current_scores"][layer_id], 3051 label=f"Current:{self.neuron_module_vector[layer_id].name}", 3052 color=color, 3053 linestyle="--", 3054 ) 3055 3056 pd2 = pd.DataFrame( 3057 { 3058 "Epochs": np.arange( 3059 len(self.member_vars["current_scores"][layer_id]) 3060 ), 3061 f"Best current for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 3062 "current_scores" 3063 ][ 3064 layer_id 3065 ], 3066 } 3067 ) 3068 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 3069 3070 plt.title(save_folder + "/" + self.save_name + " Best PBScores") 3071 plt.xlabel("Epochs") 3072 plt.ylabel("Best PBScore") 3073 ax.legend( 3074 bbox_to_anchor=(1.05, 1), 3075 loc="upper left", 3076 ncol=max(1, math.ceil(len(self.neuron_module_vector) / 30)), 3077 ) 3078 _pai_grid(ax) 3079 for switcher in self.member_vars["p_switch_epochs"]: 3080 plt.axvline( 3081 x=switcher, ymin=0, ymax=1, color=_PAI_PALETTE["dendrite"] 3082 ) 3083 3084 if self.member_vars["mode"] == "p": 3085 missed_time = ( 3086 self.member_vars["num_epochs_run"] 3087 - self.member_vars["epoch_last_improved"] 3088 ) 3089 plt.axvline( 3090 x=(len(self.member_vars["best_scores"][0]) - (missed_time + 1)), 3091 ymin=0, 3092 ymax=1, 3093 color=_PAI_PALETTE["normal"], 3094 ) 3095 3096 # pd1 here will be none if no PB layers are created 3097 if pd1 is not None: 3098 pd1.to_csv( 3099 save_folder 3100 + "/" 3101 + self.save_name 3102 + extra_string 3103 + "Best PBScores.csv", 3104 index=False, 3105 ) 3106 del pd1, pd2 3107 3108 def generate_extra_csv_files(self, save_folder, extra_string): 3109 """ 3110 Generate additional csvs 3111 3112 Parameters 3113 ---------- 3114 save_folder : str 3115 The folder to save the plots and csvs in. 3116 extra_string : str 3117 An extra string to append to the filenames. 3118 3119 Returns 3120 ------- 3121 None 3122 3123 """ 3124 pd1 = pd.DataFrame( 3125 { 3126 "Switch Number": np.arange(len(self.member_vars["switch_epochs"])), 3127 "Switch Epoch": self.member_vars["switch_epochs"], 3128 } 3129 ) 3130 pd1.to_csv( 3131 save_folder + "/" + self.save_name + extra_string + "switch_epochs.csv", 3132 index=False, 3133 ) 3134 del pd1 3135 3136 pd1 = pd.DataFrame( 3137 { 3138 "Switch Number": np.arange(len(self.member_vars["param_counts"])), 3139 "Param Count": self.member_vars["param_counts"], 3140 } 3141 ) 3142 pd1.to_csv( 3143 save_folder + "/" + self.save_name + extra_string + "param_counts.csv", 3144 index=False, 3145 ) 3146 del pd1 3147 3148 """ 3149 Create best_arch_scores.csv file 3150 When working with dendrites there is a tradeoff between additional param count and score improvement. 3151 This file will help track that tradeoff by recording the best scores for all extra_scores 3152 and extra_scores_without_graphing for each architecture version. 3153 The scores recorded here are from the epoch when the best validation score was found 3154 within each switch_epoch boundary. 3155 """ 3156 switch_counts = len(self.member_vars["switch_epochs"]) 3157 best_valid = [] 3158 associated_params = [] 3159 3160 # Initialize dictionaries to store best scores for each extra score type 3161 best_extra_scores = {} 3162 for score_name in self.member_vars["extra_scores"]: 3163 best_extra_scores[score_name] = [] 3164 for score_name in self.member_vars["extra_scores_without_graphing"]: 3165 best_extra_scores[score_name] = [] 3166 3167 for switch in range(0, switch_counts, 2): 3168 start_index = 0 3169 if switch != 0: 3170 start_index = self.member_vars["switch_epochs"][switch - 1] + 1 3171 end_index = self.member_vars["switch_epochs"][switch] + 1 3172 3173 if GPA.pai_tracker.member_vars["maximizing_score"]: 3174 best_valid_index = start_index + np.argmax( 3175 self.member_vars["accuracies"][start_index:end_index] 3176 ) 3177 else: 3178 best_valid_index = start_index + np.argmin( 3179 self.member_vars["accuracies"][start_index:end_index] 3180 ) 3181 3182 best_valid_score = self.member_vars["accuracies"][best_valid_index] 3183 best_valid.append(best_valid_score) 3184 3185 # Get corresponding scores from all extra_scores 3186 for score_name in self.member_vars["extra_scores"]: 3187 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 3188 best_extra_scores[score_name].append( 3189 self.member_vars["extra_scores"][score_name][best_valid_index] 3190 ) 3191 else: 3192 best_extra_scores[score_name].append(None) 3193 3194 # Get corresponding scores from all extra_scores_without_graphing 3195 for score_name in self.member_vars["extra_scores_without_graphing"]: 3196 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 3197 best_extra_scores[score_name].append( 3198 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 3199 ) 3200 else: 3201 best_extra_scores[score_name].append(None) 3202 3203 if self.member_vars["doing_pai"]: 3204 associated_params.append(self.member_vars["param_counts"][switch]) 3205 else: 3206 associated_params.append(self.member_vars["param_counts"][-1]) 3207 3208 # If in neuron training mode but not the very first epoch 3209 if self.member_vars["mode"] == "n" and ( 3210 (len(self.member_vars["switch_epochs"]) == 0) 3211 or ( 3212 self.member_vars["switch_epochs"][-1] + 1 3213 != len(self.member_vars["accuracies"]) 3214 ) 3215 ): 3216 start_index = 0 3217 if len(self.member_vars["switch_epochs"]) != 0: 3218 start_index = self.member_vars["switch_epochs"][-1] + 1 3219 3220 if GPA.pai_tracker.member_vars["maximizing_score"]: 3221 best_valid_index = start_index + np.argmax( 3222 self.member_vars["accuracies"][start_index:] 3223 ) 3224 else: 3225 best_valid_index = start_index + np.argmin( 3226 self.member_vars["accuracies"][start_index:] 3227 ) 3228 3229 best_valid_score = self.member_vars["accuracies"][best_valid_index] 3230 best_valid.append(best_valid_score) 3231 3232 # Get corresponding scores from all extra_scores 3233 for score_name in self.member_vars["extra_scores"]: 3234 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 3235 best_extra_scores[score_name].append( 3236 self.member_vars["extra_scores"][score_name][best_valid_index] 3237 ) 3238 else: 3239 best_extra_scores[score_name].append(None) 3240 3241 # Get corresponding scores from all extra_scores_without_graphing 3242 for score_name in self.member_vars["extra_scores_without_graphing"]: 3243 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 3244 best_extra_scores[score_name].append( 3245 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 3246 ) 3247 else: 3248 best_extra_scores[score_name].append(None) 3249 3250 associated_params.append(self.member_vars["param_counts"][-1]) 3251 3252 # Build dataframe with all columns 3253 csv_data = { 3254 "Param Counts": associated_params, 3255 "Max Valid Scores": best_valid, 3256 } 3257 3258 # Add columns for each extra score 3259 for score_name in best_extra_scores: 3260 csv_data[score_name] = best_extra_scores[score_name] 3261 3262 pd1 = pd.DataFrame(csv_data) 3263 pd1.to_csv( 3264 save_folder + "/" + self.save_name + extra_string + "_best_arch_scores.csv", 3265 index=False, 3266 ) 3267 del pd1 3268 3269 def save_graphs(self, extra_string=""): 3270 """ 3271 Save graphs and csvs for all the values the tracker records 3272 3273 Parameters 3274 ---------- 3275 extra_string : str 3276 An extra string to append to the filenames. 3277 3278 Returns 3279 ------- 3280 None 3281 3282 """ 3283 # If running DDP only save with rank 0 3284 if "RANK" in os.environ: 3285 if int(os.environ["RANK"]) != 0: 3286 return 3287 if not self.making_graphs: 3288 return 3289 3290 save_folder = "./" + self.save_name + "/" 3291 3292 plt.ioff() 3293 fig = plt.figure(figsize=(28, 14)) 3294 3295 # Plot with accuracy scores 3296 ax = plt.subplot(221) 3297 self.generate_accuracy_plots(ax, save_folder, extra_string) 3298 3299 # Plot dendrite learning scores 3300 ax = plt.subplot(222) 3301 self.generate_dendrite_learning_plots(ax, save_folder, extra_string) 3302 3303 if GPA.pc.get_drawing_extra_graphs(): 3304 # Plot learning rates for each training epoch 3305 ax = plt.subplot(223) 3306 self.generate_learning_rate_plots(ax, save_folder, extra_string) 3307 3308 # Plot the times for each training epoch 3309 ax = plt.subplot(224) 3310 self.generate_time_plots(ax, save_folder, extra_string) 3311 3312 # Generate extra CSV files 3313 self.generate_extra_csv_files(save_folder, extra_string) 3314 3315 fig.tight_layout() 3316 plt.savefig(save_folder + "/" + self.save_name + extra_string + ".png") 3317 plt.close("all") 3318 3319 def add_loss(self, loss): 3320 """Add loss to tracking vectors. 3321 3322 Parameters 3323 ---------- 3324 loss : float or int 3325 The loss value to add. 3326 3327 Returns 3328 ------- 3329 None 3330 3331 """ 3332 if not isinstance(loss, (float, int)): 3333 loss = loss.item() 3334 self.member_vars["training_loss"].append(loss) 3335 3336 def add_learning_rate(self, learning_rate): 3337 """Add learning rate to tracking vectors. 3338 3339 Parameters 3340 ---------- 3341 learning_rate : float or int 3342 The learning rate value to add. 3343 3344 Returns 3345 ------- 3346 None 3347 3348 """ 3349 if not isinstance(learning_rate, (float, int)): 3350 learning_rate = learning_rate.item() 3351 self.member_vars["training_learning_rates"].append(learning_rate) 3352 3353 def add_extra_score(self, score, extra_score_name): 3354 """Add extra score to tracking vectors. 3355 3356 Parameters 3357 ---------- 3358 score : float or int 3359 The score value to add. 3360 3361 extra_score_name : str 3362 The name of the extra score. 3363 3364 Returns 3365 ------- 3366 None 3367 3368 """ 3369 if not isinstance(score, (float, int)): 3370 try: 3371 score = score.item() 3372 except: 3373 print( 3374 "Scores added for Perforated Backpropagation should be " 3375 "float, int, or tensor, yours is a:" 3376 ) 3377 print(type(score)) 3378 pdb.set_trace() 3379 3380 if GPA.pc.get_verbose(): 3381 print(f"Adding extra score {extra_score_name} of {float(score)}") 3382 3383 if extra_score_name not in self.member_vars["extra_scores"]: 3384 self.member_vars["extra_scores"][extra_score_name] = [] 3385 self.member_vars["extra_scores"][extra_score_name].append(score) 3386 3387 if self.member_vars["mode"] == "n": 3388 if extra_score_name not in self.member_vars["n_extra_scores"]: 3389 self.member_vars["n_extra_scores"][extra_score_name] = [] 3390 self.member_vars["n_extra_scores"][extra_score_name].append(score) 3391 3392 def add_extra_score_without_graphing(self, score, extra_score_name): 3393 """Add extra score without graphing to tracking vectors. 3394 3395 Parameters 3396 ---------- 3397 score : float or int 3398 The score value to add. 3399 3400 extra_score_name : str 3401 The name of the extra score. 3402 3403 Returns 3404 ------- 3405 None 3406 3407 """ 3408 if not isinstance(score, (float, int)): 3409 try: 3410 score = score.item() 3411 except: 3412 print( 3413 "Scores added for Perforated Backpropagation should be " 3414 "float, int, or tensor, yours is a:" 3415 ) 3416 print(type(score)) 3417 print("in add_extra_score_without_graphing") 3418 pdb.set_trace() 3419 3420 if GPA.pc.get_verbose(): 3421 print(f"Adding extra score {extra_score_name} of {float(score)}") 3422 3423 if extra_score_name not in self.member_vars["extra_scores_without_graphing"]: 3424 self.member_vars["extra_scores_without_graphing"][extra_score_name] = [] 3425 self.member_vars["extra_scores_without_graphing"][extra_score_name].append( 3426 score 3427 ) 3428 3429 def add_test_score(self, score, extra_score_name): 3430 """Add test score to tracking vectors. 3431 3432 Parameters 3433 ---------- 3434 score : float or int 3435 The score value to add. 3436 3437 extra_score_name : str 3438 The name of the extra score. 3439 3440 Returns 3441 ------- 3442 None 3443 3444 Notes 3445 ----- 3446 This function is a wrapper around `add_extra_score` that separates 3447 test score for adding to best_arch_scores.csv. 3448 3449 """ 3450 self.add_extra_score(score, extra_score_name) 3451 3452 if not isinstance(score, (float, int)): 3453 try: 3454 score = score.item() 3455 except: 3456 print( 3457 "Scores added for Perforated Backpropagation should be " 3458 "float, int, or tensor, yours is a:" 3459 ) 3460 print(type(score)) 3461 print("in add_test_score") 3462 pdb.set_trace() 3463 3464 if GPA.pc.get_verbose(): 3465 print(f"Adding test score {extra_score_name} of {float(score)}") 3466 self.member_vars["test_scores"].append(score) 3467 3468 def add_validation_score(self, accuracy, net, force_switch=False): 3469 """Function to add the validation score. 3470 3471 This is complex because it determines neuron and dendrite switching. 3472 3473 Parameters 3474 ---------- 3475 accuracy : float or int 3476 The accuracy or loss value to add. 3477 net : object 3478 The neural network model. 3479 force_switch : bool, optional 3480 Whether to force a switch, by default False. 3481 3482 Returns 3483 ------- 3484 net : object 3485 The potentially modified neural network model. 3486 training_complete : bool 3487 Whether training is complete. 3488 restructured : bool 3489 Whether the model has been restructured. 3490 3491 Notes 3492 ----- 3493 WARNING: Do not call self anywhere in this function. When systems 3494 get loaded the actual tracker you are working with can change. 3495 """ 3496 3497 _pai_log("info", f"Adding validation score {accuracy:.8f}") 3498 3499 update_learning_rate() 3500 update_param_count(net) 3501 3502 accuracy = check_input_problems(net, accuracy) 3503 3504 if len(GPA.pai_tracker.member_vars["switch_epochs"]) == 0: 3505 epochs_since_cycle_switch = GPA.pai_tracker.member_vars["num_epochs_run"] 3506 else: 3507 epochs_since_cycle_switch = ( 3508 GPA.pai_tracker.member_vars["num_epochs_run"] 3509 - GPA.pai_tracker.member_vars["switch_epochs"][-1] 3510 ) 3511 3512 update_running_accuracy(accuracy, epochs_since_cycle_switch) 3513 if GPA.pc.get_perforated_backpropagation(): 3514 TPB.update_pb_scores(self) 3515 3516 # Captured before any switch below flips the mode and reloads scores. 3517 # epoch_phase is sampled here too so the emitted epoch event's phase and 3518 # its pb_scores always describe the same side of a switch boundary. 3519 epoch_phase = GPA.pai_tracker.member_vars["mode"] 3520 epoch_pb_scores = self.get_current_pb_scores() 3521 epoch_pb_scores_current = self.get_current_pb_scores_current() 3522 3523 GPA.pai_tracker.stop_epoch(internal_call=True) 3524 3525 # If it is neuron training mode 3526 if ( 3527 GPA.pai_tracker.member_vars["mode"] == "n" 3528 or GPA.pc.get_learn_dendrites_live() 3529 ): 3530 check_new_best(net, accuracy, epochs_since_cycle_switch) 3531 elif GPA.pc.get_perforated_backpropagation(): 3532 TPB.check_best_pai_score_improvement() 3533 3534 # Save the latest model 3535 if GPA.pc.get_test_saves(): 3536 UPA.save_system(net, GPA.pc.get_save_name(), "latest") 3537 if GPA.pc.get_pai_saves(): 3538 UPA.pai_save_system(net, GPA.pc.get_save_name(), "latest") 3539 3540 restructuring_status_value = NO_MODEL_UPDATE 3541 # If it is time to switch based on scores and counter or a manual switch 3542 if GPA.pai_tracker.switch_time() or force_switch: 3543 # If testing dendrite capacity switch after enough dendrites added 3544 if ( 3545 (GPA.pai_tracker.member_vars["mode"] == "n") 3546 and (GPA.pai_tracker.member_vars["num_dendrites_added"] > 2) 3547 and GPA.pc.get_testing_dendrite_capacity() 3548 ): 3549 GPA.pai_tracker.save_graphs() 3550 _pai_log( 3551 "info", 3552 "Successfully added 3 dendrites with GPA.pc.set_testing_dendrite_capacity(True) (default). " 3553 "You may now set that to False and run a real experiment.", 3554 ) 3555 return net, False, True 3556 3557 # If doing neuron training but this dendrite count didn't improve 3558 if ( 3559 (GPA.pai_tracker.member_vars["mode"] == "n") 3560 or GPA.pc.get_learn_dendrites_live() 3561 ) and (GPA.pai_tracker.member_vars["current_n_set_global_best"] is False): 3562 new_restructuring_status_value, net = process_no_improvement(net) 3563 # if this was the final try return that training is complete 3564 if new_restructuring_status_value == TRAINING_COMPLETE: 3565 if _dashboard_emitter is not None: 3566 _dashboard_emitter.emit_run_end( 3567 GPA.pc, 3568 epoch_last_improved=GPA.pai_tracker.member_vars[ 3569 "epoch_last_improved" 3570 ], 3571 global_best_score=GPA.pai_tracker.member_vars[ 3572 "global_best_validation_score" 3573 ], 3574 ) 3575 return net, True, True 3576 else: 3577 restructuring_status_value = update_restructuring_status( 3578 restructuring_status_value, new_restructuring_status_value 3579 ) 3580 # Else if did improve, do a normal switch process 3581 else: 3582 if GPA.pc.get_verbose(): 3583 print( 3584 f"Calling switch_mode with " 3585 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]}, ' 3586 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 3587 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]}, ' 3588 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_value"]},' 3589 f'{GPA.pc.get_max_dendrites()},' 3590 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]},' 3591 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"]},' 3592 ) 3593 import pdb; pdb.set_trace 3594 # If the max number of dendrites has been hit or not doing pai and adding dendtites 3595 # then return rather than adding more 3596 if ( 3597 (GPA.pai_tracker.member_vars["mode"] == "n") 3598 and ( 3599 GPA.pc.get_max_dendrites() 3600 == GPA.pai_tracker.member_vars["num_dendrites_added"] 3601 ) 3602 ) or (GPA.pai_tracker.member_vars["doing_pai"] is False): 3603 if GPA.pc.get_verbose(): 3604 print( 3605 "Max dendrites reached or not doing PAI, finishing training" 3606 ) 3607 net = process_final_network(net) 3608 # Increment integrated if we have dendrites (means they're integrated) 3609 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3610 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3611 _pai_log("info", f"Final dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3612 if _dashboard_emitter is not None: 3613 _dashboard_emitter.emit_dendrite_added( 3614 GPA.pc, 3615 epoch_index=len( 3616 GPA.pai_tracker.member_vars["accuracies"] 3617 ) - 1, 3618 num_dendrites_integrated=GPA.pai_tracker.member_vars[ 3619 "num_dendrites_integrated" 3620 ], 3621 param_count=UPA.count_params(net), 3622 ) 3623 if _dashboard_emitter is not None: 3624 _dashboard_emitter.emit_run_end( 3625 GPA.pc, 3626 epoch_last_improved=GPA.pai_tracker.member_vars[ 3627 "epoch_last_improved" 3628 ], 3629 global_best_score=GPA.pai_tracker.member_vars[ 3630 "global_best_validation_score" 3631 ], 3632 ) 3633 return net, True, True 3634 3635 # Otherwise if its neuron training mode reset the counter of failed dendrites 3636 # Check if we should increment integrated count BEFORE change_learning_modes loads old state 3637 should_increment_integrated = False 3638 if GPA.pai_tracker.member_vars["mode"] == "n": 3639 GPA.pai_tracker.member_vars["num_dendrite_tries"] = 0 3640 if GPA.pc.get_verbose(): 3641 print( 3642 "Adding new dendrites without resetting which means " 3643 "the last ones improved. Resetting num_dendrite_tries" 3644 ) 3645 # Remember to increment after change_learning_modes (which loads old tracker state) 3646 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3647 should_increment_integrated = True 3648 3649 GPA.pai_tracker.save_graphs( 3650 f'_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}' 3651 ) 3652 3653 if GPA.pc.get_test_saves(): 3654 UPA.save_system( 3655 net, 3656 GPA.pc.get_save_name(), 3657 f'beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3658 ) 3659 # Copy current best model from this set of dendrites 3660 # If running DDP only copy with rank 0 3661 if "RANK" not in os.environ or int(os.environ["RANK"]) == 0: 3662 shutil.copyfile( 3663 f"{GPA.pc.get_save_name()}/best_model.pt", 3664 f'{GPA.pc.get_save_name()}/best_model_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}.pt', 3665 ) 3666 3667 if GPA.pc.get_dashboard_debug(): 3668 _pai_log( 3669 "debug", 3670 "[emitter/finding4] before change_learning_modes: " 3671 f'len(accuracies)={len(GPA.pai_tracker.member_vars["accuracies"])} ' 3672 f'num_epochs_run={GPA.pai_tracker.member_vars["num_epochs_run"]}', 3673 ) 3674 net = UPA.change_learning_modes( 3675 net, 3676 GPA.pc.get_save_name(), 3677 "best_model", 3678 GPA.pai_tracker.member_vars["doing_pai"], 3679 ) 3680 restructuring_status_value = NETWORK_RESTRUCTURED 3681 if GPA.pc.get_dashboard_debug(): 3682 _pai_log( 3683 "debug", 3684 "[emitter/finding4] after change_learning_modes: " 3685 f'len(accuracies)={len(GPA.pai_tracker.member_vars["accuracies"])} ' 3686 f'num_epochs_run={GPA.pai_tracker.member_vars["num_epochs_run"]}', 3687 ) 3688 3689 # Now increment after change_learning_modes has loaded the best model 3690 # This ensures the increment persists and doesn't get overwritten 3691 if should_increment_integrated: 3692 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3693 _pai_log("info", f"Dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3694 if _dashboard_emitter is not None: 3695 _dashboard_emitter.emit_dendrite_added( 3696 GPA.pc, 3697 epoch_index=len( 3698 GPA.pai_tracker.member_vars["accuracies"] 3699 ) - 1, 3700 num_dendrites_integrated=GPA.pai_tracker.member_vars[ 3701 "num_dendrites_integrated" 3702 ], 3703 param_count=UPA.count_params(net), 3704 ) 3705 3706 # If restructured is true, clear scheduler/optimizer before saving 3707 if restructuring_status_value != NETWORK_RESTRUCTURED: 3708 print( 3709 "Restructured should always be triggered here, let us know if you encounter this situation" 3710 ) 3711 pdb.set_trace() 3712 3713 # Since there is a restructuring optimizer and scheduler must be reinitialized after return 3714 GPA.pai_tracker.clear_optimizer_and_scheduler() 3715 3716 # Save the model from after the switch 3717 UPA.save_system( 3718 net, 3719 GPA.pc.get_save_name(), 3720 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3721 ) 3722 3723 # If not time to switch and you have a scheduler, perform the update step 3724 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 3725 new_restructuring_status_value, net = process_scheduler_update( 3726 net, accuracy, epochs_since_cycle_switch 3727 ) 3728 restructuring_status_value = update_restructuring_status( 3729 restructuring_status_value, new_restructuring_status_value 3730 ) 3731 3732 GPA.pai_tracker.start_epoch(internal_call=True) 3733 if _dashboard_emitter is not None: 3734 _mv = GPA.pai_tracker.member_vars 3735 _lr = _mv["training_learning_rates"][-1] if _mv["training_learning_rates"] else None 3736 # epoch_index is the committed coordinate: len(accuracies) - 1, 3737 # captured after update_running_accuracy appends and after any reload 3738 # truncation in switch handling. Independent of num_epochs_run. 3739 _epoch_index = len(_mv["accuracies"]) - 1 3740 if GPA.pc.get_dashboard_debug(): 3741 _pai_log( 3742 "debug", 3743 "[emitter/finding4] emit_epoch: " 3744 f"epoch_index={_epoch_index} " 3745 f'true_epoch={_mv["num_epochs_run"]} ' 3746 f'restructured={restructuring_status_value == NETWORK_RESTRUCTURED} ' 3747 f'tail_accuracies={_mv["accuracies"][-4:]}', 3748 ) 3749 # Phase captured at the top of the method (pre-switch), matching the 3750 # point pb_scores were sampled - see Finding 2, ticket 06. 3751 _phase = epoch_phase 3752 # scores: validation (reported), running (checkpointed member var), 3753 # then every add_extra_score name verbatim. 3754 _scores = { 3755 "validation": accuracy, 3756 "running": _mv["running_accuracies"][-1] if _mv["running_accuracies"] else None, 3757 } 3758 for _name, _vals in _mv["extra_scores"].items(): 3759 if _vals: 3760 _scores[_name] = _vals[-1] 3761 _n_times = _mv["n_epoch_times"] or [(_mv["n_train_times"][-1] + _mv["n_val_times"][-1]) if (_mv["n_train_times"] and _mv["n_val_times"]) else None] 3762 _p_times = _mv["p_epoch_times"] or [(_mv["p_train_times"][-1] + _mv["p_val_times"][-1]) if (_mv["p_train_times"] and _mv["p_val_times"]) else None] 3763 _dashboard_emitter.emit_epoch( 3764 GPA.pc, 3765 epoch_index=_epoch_index, 3766 phase=_phase, 3767 scores=_scores, 3768 learning_rate=_lr, 3769 normal_time=_n_times[-1] if _phase == "n" else None, 3770 pai_time=_p_times[-1] if _phase == "p" else None, 3771 pb_scores=epoch_pb_scores, 3772 pb_scores_current=epoch_pb_scores_current, 3773 ) 3774 GPA.pai_tracker.save_graphs() 3775 3776 if restructuring_status_value == NETWORK_RESTRUCTURED: 3777 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 3778 GPA.pai_tracker.member_vars["num_epochs_run"] 3779 ) 3780 if GPA.pc.get_verbose(): 3781 print( 3782 f"Setting epoch last improved to " 3783 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]}' 3784 ) 3785 3786 now = datetime.now() 3787 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 3788 3789 if GPA.pc.get_verbose(): 3790 print("Not saving restructure right now") 3791 3792 """ 3793 This block of code helped with a save issue with safetensors and huggingface, but it breaks DDP. 3794 Temporarily removing it to avoid DDP issues, but if you encounter save issues try adding it back in. 3795 for param in net.parameters(): 3796 param.data = param.data.contiguous() 3797 """ 3798 if GPA.pc.get_verbose(): 3799 print( 3800 f"Completed adding score. Restructured is {restructuring_status_value}, " 3801 f"\ncurrent switch list is:" 3802 ) 3803 print(GPA.pai_tracker.member_vars["switch_epochs"]) 3804 3805 if _dashboard_emitter is not None and restructuring_status_value == NETWORK_RESTRUCTURED: 3806 _param_count = UPA.count_params(net) 3807 _dashboard_emitter.emit_switch( 3808 GPA.pc, 3809 # switch_epochs was already appended in change_learning_modes, 3810 # so subtract 1 for a 0-based ordinal (matches matplotlib's 3811 # np.arange(len(switch_epochs)) labelling) - Finding 1, ticket 06. 3812 switch_ordinal=len(GPA.pai_tracker.member_vars["switch_epochs"]) - 1, 3813 epoch_index=len(GPA.pai_tracker.member_vars["accuracies"]) - 1, 3814 param_count=_param_count, 3815 switch_type=GPA.pai_tracker.member_vars["mode"], 3816 ) 3817 3818 # Always False for training complete if nothing triggered that training is over 3819 return net, restructuring_status_value, False 3820 3821 def clear_all_processors(self): 3822 """Clear all processors from modules. 3823 3824 Parameters 3825 ---------- 3826 None 3827 3828 Returns 3829 ------- 3830 None 3831 This function does not return a value. 3832 """ 3833 for module in self.neuron_module_vector: 3834 module.clear_processors() 3835 3836 def create_new_dendrite_module(self): 3837 """Add dendrite module to all neuron modules. 3838 3839 Parameters 3840 ---------- 3841 None 3842 3843 Returns 3844 ------- 3845 None 3846 This function does not return a value. 3847 """ 3848 for module in self.neuron_module_vector: 3849 module.create_new_dendrite_module() 3850 3851 def set_create_dendrite_global(self, fn): 3852 """Call set_create_dendrite(fn) on every tracked PAINeuronModule.""" 3853 for module in self.neuron_module_vector: 3854 module.set_create_dendrite(fn) 3855 3856 def set_dendrite_loss_fn_global(self, fn): 3857 """Set the global dendrite loss function used by all dendrite modules.""" 3858 from perforatedbp import modules_pbp as MPB 3859 MPB.dendrite_loss_fn = fn 3860 3861 def apply_pb_grads(self): 3862 """Apply perforated backpropagation gradients to all modules. 3863 3864 Parameters 3865 ---------- 3866 None 3867 3868 Returns 3869 ------- 3870 None 3871 This function does not return a value. 3872 """ 3873 if self.member_vars["mode"] == "p": 3874 for module in self.neuron_module_vector: 3875 module.apply_pb_grads() 3876 3877 def apply_pb_zero(self): 3878 """Apply perforated backpropagation zero gradients to all modules. 3879 3880 Parameters 3881 ---------- 3882 None 3883 3884 Returns 3885 ------- 3886 None 3887 This function does not return a value. 3888 """ 3889 if self.member_vars["mode"] == "p": 3890 for module in self.neuron_module_vector: 3891 module.apply_pb_zero()
NO_MODEL_UPDATE =
0
NETWORK_RESTRUCTURED =
1
TRAINING_COMPLETE =
2
STEP_CLEARED =
0
STEP_CALLED =
1
def
update_restructuring_status(old_status, new_status):
102def update_restructuring_status(old_status, new_status): 103 """Update restructured variable during add_validation_score 104 105 Update the restructuring status based on the new status. 106 If the new status is that there was not an update, 107 dont overwrite the old status which may show there was an update. 108 109 Parameters 110 ---------- 111 old_status : int 112 The old restructuring status. 113 new_status : int 114 The new restructuring status. 115 116 Returns 117 ------- 118 int 119 The updated restructuring status. 120 121 """ 122 if new_status == NETWORK_RESTRUCTURED or new_status == TRAINING_COMPLETE: 123 return NETWORK_RESTRUCTURED 124 else: 125 return old_status
Update restructured variable during add_validation_score
Update the restructuring status based on the new status. If the new status is that there was not an update, dont overwrite the old status which may show there was an update.
Parameters
- old_status (int): The old restructuring status.
- new_status (int): The new restructuring status.
Returns
- int: The updated restructuring status.
def
update_learning_rate():
128def update_learning_rate(): 129 """Update the learning rate in the tracker. 130 131 Parameters 132 ---------- 133 None 134 135 Returns 136 ------- 137 None 138 This function does not return a value. 139 """ 140 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 141 learning_rate = param_group["lr"] 142 GPA.pai_tracker.add_learning_rate(learning_rate)
Update the learning rate in the tracker.
Parameters
- None
Returns
- None: This function does not return a value.
def
update_param_count(net):
145def update_param_count(net): 146 """Update the parameter count in the tracker if not already set. 147 148 Parameters 149 ---------- 150 net : torch.nn.Module 151 The neural network model to count parameters for. 152 Returns 153 ------- 154 None 155 """ 156 if len(GPA.pai_tracker.member_vars["param_counts"]) == 0: 157 GPA.pai_tracker.member_vars["param_counts"].append(UPA.count_params(net))
Update the parameter count in the tracker if not already set.
Parameters
- net (torch.nn.Module): The neural network model to count parameters for.
Returns
- None
def
check_input_problems(net, accuracy):
160def check_input_problems(net, accuracy): 161 """Check for potential input problems in add_validation_score. 162 163 Parameters 164 ---------- 165 net : torch.nn.Module 166 The neural network model to check. 167 accuracy : float, int, or torch.Tensor 168 The accuracy score to validate. 169 170 Returns 171 ------- 172 float 173 The validated accuracy score. 174 175 """ 176 177 # Make sure you are passing in the model and not the dataparallel wrapper 178 if issubclass(type(net), nn.DataParallel): 179 _pai_log("error", "Need to call .module when using add validation score") 180 pdb.set_trace() 181 sys.exit(-1) 182 183 if "module" in net.__dir__(): 184 _pai_log("error", "Need to call .module when using add validation score") 185 pdb.set_trace() 186 sys.exit(-1) 187 188 if not isinstance(accuracy, (float, int)): 189 try: 190 accuracy = accuracy.item() 191 except: 192 _pai_log( 193 "error", 194 f"Scores added for add_validation_score should be float, int, or tensor, yours is a: {type(accuracy)}", 195 ) 196 pdb.set_trace() 197 sys.exit(-1) 198 return accuracy
Check for potential input problems in add_validation_score.
Parameters
- net (torch.nn.Module): The neural network model to check.
- accuracy (float, int, or torch.Tensor): The accuracy score to validate.
Returns
- float: The validated accuracy score.
def
update_running_accuracy(accuracy, epochs_since_cycle_switch):
201def update_running_accuracy(accuracy, epochs_since_cycle_switch): 202 """Add the new accuracy to the tracker. 203 204 Parameters 205 ---------- 206 accuracy : float, int, or torch.Tensor 207 The accuracy score to add. 208 epochs_since_cycle_switch : int 209 The number of epochs since the last cycle switch. 210 211 Returns 212 ------- 213 None 214 215 """ 216 # Only update running_accuracy when neurons are being updated 217 if GPA.pai_tracker.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 218 if epochs_since_cycle_switch < GPA.pc.get_initial_history_after_switches(): 219 if epochs_since_cycle_switch <= 0: 220 GPA.pai_tracker.member_vars["running_accuracy"] = accuracy 221 else: 222 GPA.pai_tracker.member_vars[ 223 "running_accuracy" 224 ] = GPA.pai_tracker.member_vars["running_accuracy"] * ( 225 1 - (1.0 / (epochs_since_cycle_switch + 1)) 226 ) + accuracy * ( 227 1.0 / (epochs_since_cycle_switch + 1) 228 ) 229 else: 230 GPA.pai_tracker.member_vars[ 231 "running_accuracy" 232 ] = GPA.pai_tracker.member_vars["running_accuracy"] * ( 233 1.0 - 1.0 / GPA.pc.get_history_lookback() 234 ) + accuracy * ( 235 1.0 / GPA.pc.get_history_lookback() 236 ) 237 238 GPA.pai_tracker.member_vars["accuracies"].append(accuracy) 239 if GPA.pai_tracker.member_vars["mode"] == "n": 240 GPA.pai_tracker.member_vars["n_accuracies"].append(accuracy) 241 242 if ( 243 GPA.pc.get_drawing_pai() 244 or GPA.pai_tracker.member_vars["mode"] == "n" 245 or GPA.pc.get_learn_dendrites_live() 246 ): 247 GPA.pai_tracker.member_vars["running_accuracies"].append( 248 GPA.pai_tracker.member_vars["running_accuracy"] 249 )
Add the new accuracy to the tracker.
Parameters
- accuracy (float, int, or torch.Tensor): The accuracy score to add.
- epochs_since_cycle_switch (int): The number of epochs since the last cycle switch.
Returns
- None
def
score_beats_current_best(new_score, old_score):
252def score_beats_current_best(new_score, old_score): 253 """Check if the new score beats the current best score. 254 255 Parameters 256 ---------- 257 new_score : float 258 The new score to compare. 259 old_score : float 260 The old score to compare against. 261 262 Returns 263 ------- 264 bool 265 True if the new score beats the old score, False otherwise. 266 267 Notes 268 ----- 269 Must beat the old score by the margins set in globals for improvement thresholds. 270 271 """ 272 return ( 273 GPA.pai_tracker.member_vars["maximizing_score"] 274 and (new_score * (1.0 - GPA.pc.get_improvement_threshold()) > old_score) 275 and new_score - GPA.pc.get_improvement_threshold_raw() > old_score 276 ) or ( 277 (not GPA.pai_tracker.member_vars["maximizing_score"]) 278 and (new_score * (1.0 + GPA.pc.get_improvement_threshold()) < old_score) 279 and (new_score + GPA.pc.get_improvement_threshold_raw()) < old_score 280 )
Check if the new score beats the current best score.
Parameters
- new_score (float): The new score to compare.
- old_score (float): The old score to compare against.
Returns
- bool: True if the new score beats the old score, False otherwise.
Notes
Must beat the old score by the margins set in globals for improvement thresholds.
def
check_new_best(net, accuracy, epochs_since_cycle_switch):
283def check_new_best(net, accuracy, epochs_since_cycle_switch): 284 """Check if the new accuracy is a new best. 285 286 Performs saves if new best score is found. 287 288 Parameters 289 ---------- 290 net : torch.nn.Module 291 The neural network model being trained. 292 accuracy : float 293 The accuracy score to check. 294 epochs_since_cycle_switch : int 295 The number of epochs since the last cycle switch. 296 297 Returns 298 ------- 299 None 300 301 """ 302 score_improved = score_beats_current_best( 303 GPA.pai_tracker.member_vars["running_accuracy"], 304 GPA.pai_tracker.member_vars["current_best_validation_score"], 305 ) 306 307 enough_time = ( 308 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 309 ) or (GPA.pai_tracker.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME) 310 311 if ( 312 score_improved 313 or GPA.pai_tracker.member_vars["current_best_validation_score"] == 0 314 ) and enough_time: 315 316 if GPA.pai_tracker.member_vars["maximizing_score"]: 317 if GPA.pc.get_verbose(): 318 print( 319 f"\n\nGot score of {accuracy:.10f} " 320 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 321 f"*{1-GPA.pc.get_improvement_threshold()}=" 322 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 - GPA.pc.get_improvement_threshold())}) ' 323 f'which is higher than {GPA.pai_tracker.member_vars["current_best_validation_score"]:.10f} ' 324 f"by {GPA.pc.get_improvement_threshold_raw()} so setting epoch to " 325 f'{GPA.pai_tracker.member_vars["num_epochs_run"]}\n\n' 326 ) 327 else: 328 if GPA.pc.get_verbose(): 329 print( 330 f"\n\nGot score of {accuracy:.10f} " 331 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 332 f"*{1+GPA.pc.get_improvement_threshold()}=" 333 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 + GPA.pc.get_improvement_threshold())}) ' 334 f'which is lower than {GPA.pai_tracker.member_vars["current_best_validation_score"]:.10f} ' 335 f'so setting epoch to {GPA.pai_tracker.member_vars["num_epochs_run"]}\n\n' 336 ) 337 338 # Set the new best score 339 GPA.pai_tracker.member_vars["current_best_validation_score"] = ( 340 GPA.pai_tracker.member_vars["running_accuracy"] 341 ) 342 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 343 GPA.pai_tracker.member_vars["num_epochs_run"] 344 ) 345 if GPA.pc.get_verbose(): 346 print( 347 f'2 epoch improved is {GPA.pai_tracker.member_vars["epoch_last_improved"]}' 348 ) 349 # Immediately update this list before saving so loading will have it correctly 350 GPA.pai_tracker.member_vars["last_improved_accuracies"].append( 351 GPA.pai_tracker.member_vars["epoch_last_improved"] 352 ) 353 # Check if global best 354 is_global_best = score_beats_current_best( 355 GPA.pai_tracker.member_vars["current_best_validation_score"], 356 GPA.pai_tracker.member_vars["global_best_validation_score"], 357 ) 358 359 if ( 360 is_global_best 361 or GPA.pai_tracker.member_vars["global_best_validation_score"] == 0 362 ): 363 if GPA.pc.get_verbose(): 364 print( 365 f"This also beats global best of " 366 f'{GPA.pai_tracker.member_vars["global_best_validation_score"]} so saving' 367 ) 368 GPA.pai_tracker.member_vars["global_best_validation_score"] = ( 369 GPA.pai_tracker.member_vars["current_best_validation_score"] 370 ) 371 GPA.pai_tracker.member_vars["current_n_set_global_best"] = True 372 UPA.save_system(net, GPA.pc.get_save_name(), "best_model") 373 if GPA.pc.get_pai_saves(): 374 UPA.pai_save_system(net, GPA.pc.get_save_name(), "best_model") 375 else: 376 if GPA.pc.get_verbose(): 377 print("Not saving new best because:") 378 if epochs_since_cycle_switch <= GPA.pc.get_initial_history_after_switches(): 379 print( 380 f"Not enough history since switch {epochs_since_cycle_switch} <= " 381 f"{GPA.pc.get_initial_history_after_switches()}" 382 ) 383 elif GPA.pai_tracker.member_vars["maximizing_score"]: 384 print( 385 f"Got score of {accuracy} " 386 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 387 f"*{1-GPA.pc.get_improvement_threshold()}=" 388 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 - GPA.pc.get_improvement_threshold())}) ' 389 f"which is not higher than " 390 f'{GPA.pai_tracker.member_vars["current_best_validation_score"]}' 391 ) 392 else: 393 print( 394 f"Got score of {accuracy} " 395 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 396 f"*{1+GPA.pc.get_improvement_threshold()}=" 397 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 + GPA.pc.get_improvement_threshold())}) ' 398 f"which is not lower than " 399 f'{GPA.pai_tracker.member_vars["current_best_validation_score"]}' 400 ) 401 GPA.pai_tracker.member_vars["last_improved_accuracies"].append( 402 GPA.pai_tracker.member_vars["epoch_last_improved"] 403 ) 404 # If it's the first epoch, save as best anyway 405 if len(GPA.pai_tracker.member_vars["accuracies"]) == 1: 406 if GPA.pc.get_verbose(): 407 print("Saving first model or all models") 408 UPA.save_system(net, GPA.pc.get_save_name(), "best_model") 409 if GPA.pc.get_pai_saves(): 410 UPA.pai_save_system(net, GPA.pc.get_save_name(), "best_model")
Check if the new accuracy is a new best.
Performs saves if new best score is found.
Parameters
- net (torch.nn.Module): The neural network model being trained.
- accuracy (float): The accuracy score to check.
- epochs_since_cycle_switch (int): The number of epochs since the last cycle switch.
Returns
- None
def
process_no_improvement(net):
413def process_no_improvement(net): 414 """Handle the case where no improvement is observed. 415 416 If the new dendrite did not improve scores, but its time to switch modes 417 either trigger the end of learning or reset to the previous dendrite 418 to try again. 419 420 Parameters 421 ---------- 422 net : torch.nn.Module 423 The neural network model being trained. 424 425 Returns 426 ------- 427 int 428 The status of restructuring or training completion. 429 torch.nn.Module 430 The potentially modified neural network model. 431 432 """ 433 if GPA.pc.get_verbose(): 434 print( 435 f"Planning to switch to p mode but best beat last: " 436 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]} ' 437 f"current start lr steps: " 438 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 439 f"and last maximum lr steps: " 440 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 441 f'for rate: {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]:.8f}' 442 ) 443 444 now = datetime.now() 445 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 446 447 if GPA.pc.get_verbose(): 448 print( 449 f'1 saving break {dt_string}_noImprove_lr_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 450 ) 451 452 GPA.pai_tracker.save_graphs( 453 f'{dt_string}_noImprove_lr_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 454 ) 455 456 if ( 457 GPA.pai_tracker.member_vars["num_dendrite_tries"] 458 < GPA.pc.get_max_dendrite_tries() -1 459 ): 460 _pai_log( 461 "info", 462 f"The newest added dendrites did not improve but current tries " 463 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"] + 1} ' 464 f"is less than max tries {GPA.pc.get_max_dendrite_tries()} " 465 f"so loading last switch and trying new Dendrites.", 466 ) 467 old_tries = GPA.pai_tracker.member_vars["num_dendrite_tries"] 468 # Load best model from previous n mode 469 net = UPA.change_learning_modes( 470 net, 471 GPA.pc.get_save_name(), 472 "best_model", 473 GPA.pai_tracker.member_vars["doing_pai"], 474 ) 475 GPA.pai_tracker.member_vars["num_dendrite_tries"] = old_tries + 1 476 return NETWORK_RESTRUCTURED, net 477 else: 478 _pai_log( 479 "info", 480 f"The newest added dendrites did not improve system and " 481 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"] + 1} > ' 482 f"{GPA.pc.get_max_dendrite_tries()} so returning training_complete.", 483 ) 484 _pai_log("info", "You should now exit your training loop and best_model will be your final model for inference") 485 if not GPA.pc.get_perforated_backpropagation() and GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 486 _pai_log("info", "For improved results, try perforated backpropagation next time!") 487 488 old_silent = GPA.pc.get_silent() 489 GPA.pc.set_silent(True) 490 UPA.load_system(net, GPA.pc.get_save_name(), "best_model", switch_call=True) 491 GPA.pc.set_silent(old_silent) 492 GPA.pai_tracker.save_graphs() 493 UPA.pai_save_system(net, GPA.pc.get_save_name(), "final_clean") 494 return TRAINING_COMPLETE, net
Handle the case where no improvement is observed.
If the new dendrite did not improve scores, but its time to switch modes either trigger the end of learning or reset to the previous dendrite to try again.
Parameters
- net (torch.nn.Module): The neural network model being trained.
Returns
- int: The status of restructuring or training completion.
- torch.nn.Module: The potentially modified neural network model.
def
process_final_network(net):
497def process_final_network(net): 498 """When the max number of dendrites has been hit load the best_model and return 499 500 Parameters 501 ---------- 502 net : torch.nn.Module 503 The neural network model being trained. 504 505 Returns 506 ------- 507 torch.nn.Module 508 The final neural network model. 509 """ 510 511 _pai_log("info", f"Last Dendrites were good and this hit the max of {GPA.pc.get_max_dendrites()}") 512 if not GPA.pc.get_perforated_backpropagation() and GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 513 _pai_log("info", "For improved results, try perforated backpropagation next time!") 514 GPA.pai_tracker.save_graphs("before_final") 515 UPA.load_system(net, GPA.pc.get_save_name(), "best_model", switch_call=True) 516 GPA.pai_tracker.save_graphs() 517 UPA.pai_save_system(net, GPA.pc.get_save_name(), "final_clean") 518 return net
When the max number of dendrites has been hit load the best_model and return
Parameters
- net (torch.nn.Module): The neural network model being trained.
Returns
- torch.nn.Module: The final neural network model.
def
process_scheduler_update(net, accuracy, epochs_since_cycle_switch):
521def process_scheduler_update(net, accuracy, epochs_since_cycle_switch): 522 """Updates the scheduler 523 524 This increments the scheduler, but if we are automatically sweeping 525 to find the best initial learning rate for a new set of dendrites 526 this function also triggers the network at addition time to 527 try the next value. 528 529 Process for finding best initial learning rate for dendrites: 530 1. Start at default rate 531 2. Learn at that rate until scheduler increments twice 532 3. Save that version, start dendrites at LR current increment - 1 533 4. Repeat 2 and 3 until version has worse final score at set LR 534 5. Load previous model with best accuracy at that LR as initial rate 535 536 Parameters 537 ---------- 538 net : torch.nn.Module 539 The neural network model being trained. 540 accuracy : float 541 The accuracy of the model at the current learning rate. 542 epochs_since_cycle_switch : int 543 The number of epochs since the last cycle switch. 544 545 Returns 546 ------- 547 int 548 The status of restructuring or training completion. 549 torch.nn.Module 550 The potentially modified neural network model. 551 """ 552 553 restructured = False 554 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 555 learning_rate1 = param_group["lr"] 556 557 if ( 558 type(GPA.pai_tracker.member_vars["scheduler_instance"]) 559 is torch.optim.lr_scheduler.ReduceLROnPlateau 560 ): 561 if ( 562 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 563 or GPA.pai_tracker.member_vars["mode"] == "p" 564 ): 565 if GPA.pc.get_verbose(): 566 print( 567 f"Updating scheduler with last improved " 568 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]} ' 569 f'from current {GPA.pai_tracker.member_vars["num_epochs_run"]}' 570 ) 571 if GPA.pai_tracker.member_vars["scheduler"] is not None: 572 GPA.pai_tracker.member_vars["scheduler_instance"].step(metrics=accuracy) 573 if ( 574 GPA.pai_tracker.member_vars["scheduler"] 575 is torch.optim.lr_scheduler.ReduceLROnPlateau 576 ): 577 if GPA.pc.get_verbose(): 578 print( 579 f"Scheduler is now at " 580 f'{GPA.pai_tracker.member_vars["scheduler_instance"].num_bad_epochs} bad epochs' 581 ) 582 else: 583 if GPA.pc.get_verbose(): 584 print("Not stepping optimizer since hasnt initialized") 585 586 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 587 if ( 588 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 589 or GPA.pai_tracker.member_vars["mode"] == "p" 590 ): 591 if GPA.pc.get_verbose(): 592 if hasattr(GPA.pai_tracker.member_vars["scheduler_instance"], '_step_count'): 593 count = GPA.pai_tracker.member_vars["scheduler_instance"]._step_count 594 else: 595 count = GPA.pai_tracker.member_vars["scheduler_instance"].last_epoch 596 597 print( 598 f"Incrementing scheduler to count " 599 f'{count}' 600 ) 601 GPA.pai_tracker.member_vars["scheduler_instance"].step() 602 if ( 603 GPA.pai_tracker.member_vars["scheduler"] 604 is torch.optim.lr_scheduler.ReduceLROnPlateau 605 ): 606 if GPA.pc.get_verbose(): 607 print( 608 f"Scheduler is now at " 609 f'{GPA.pai_tracker.member_vars["scheduler_instance"].num_bad_epochs} bad epochs' 610 ) 611 612 if ( 613 epochs_since_cycle_switch <= GPA.pc.get_initial_history_after_switches() 614 and GPA.pai_tracker.member_vars["mode"] == "n" 615 ): 616 if GPA.pc.get_verbose(): 617 print( 618 f"Not stepping with history {GPA.pc.get_initial_history_after_switches()} " 619 f"and current {epochs_since_cycle_switch}" 620 ) 621 622 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 623 learning_rate2 = param_group["lr"] 624 625 stepped = False 626 at_last_count = False 627 628 if GPA.pc.get_verbose(): 629 print( 630 f"Checking if at last with scores " 631 f'{len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"])}, ' 632 f"count since switch {epochs_since_cycle_switch} " 633 f"and last total lr step count " 634 f'{GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"]}' 635 ) 636 637 # Check if at double or exactly the test count 638 if ( 639 len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 0 640 and epochs_since_cycle_switch 641 == GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] * 2 642 ) or ( 643 len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 1 644 and epochs_since_cycle_switch 645 == GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] 646 ): 647 at_last_count = True 648 649 if GPA.pc.get_verbose(): 650 print( 651 f"At last count {at_last_count} with count {epochs_since_cycle_switch} " 652 f'and last LR count {GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"]}' 653 ) 654 655 if learning_rate1 != learning_rate2: 656 stepped = True 657 GPA.pai_tracker.member_vars["current_step_count"] += 1 658 659 if GPA.pc.get_verbose(): 660 print( 661 f"Learning rate just stepped to {learning_rate2:.10e} " 662 f'with {GPA.pai_tracker.member_vars["current_step_count"]} total steps' 663 ) 664 665 if ( 666 GPA.pai_tracker.member_vars["current_step_count"] 667 == GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] 668 ): 669 if GPA.pc.get_verbose(): 670 print( 671 f'{GPA.pai_tracker.member_vars["current_step_count"]} ' 672 f"steps is the max of the last switch mode" 673 ) 674 # Set it when 1->2 gets to 2, not when 0->1 hits 2 as stopping point 675 if ( 676 GPA.pai_tracker.member_vars["current_step_count"] 677 - GPA.pai_tracker.member_vars[ 678 "current_n_learning_rate_initial_skip_steps" 679 ] 680 == 1 681 ): 682 GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] = ( 683 epochs_since_cycle_switch 684 ) 685 686 if GPA.pc.get_verbose(): 687 print( 688 f"Learning rates were {learning_rate1:.8e} and {learning_rate2:.8e} " 689 f'started with {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 690 f'and is now at {GPA.pai_tracker.member_vars["current_step_count"]} ' 691 f'committed {GPA.pai_tracker.member_vars["committed_to_initial_rate"]} ' 692 f"then either this (non zero) or eventually comparing to " 693 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 694 f'steps or rate {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]:.8f}' 695 ) 696 697 # If learning rate just stepped, check restart at lower rate 698 if ( 699 (GPA.pai_tracker.member_vars["scheduler"] is not None) 700 and 701 # If potentially might have higher accuracy 702 ( 703 (GPA.pai_tracker.member_vars["mode"] == "n") 704 or GPA.pc.get_learn_dendrites_live() 705 ) 706 and 707 # And learning rate just stepped 708 (stepped or at_last_count) 709 ): 710 711 # If this is the first dendrite addition (last_max_learning_rate_steps == 0), 712 # immediately commit to the initial rate without searching 713 if GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] == 0: 714 if GPA.pc.get_verbose(): 715 print( 716 f"First dendrite addition detected (last_max_learning_rate_steps == 0), " 717 f"immediately committing to initial rate without search" 718 ) 719 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 720 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 721 GPA.pai_tracker.member_vars["current_step_count"] 722 ) 723 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 724 learning_rate2 725 ) 726 727 # If hasn't committed to a learning rate for this cycle yet 728 if not GPA.pai_tracker.member_vars["committed_to_initial_rate"]: 729 best_score_so_far = GPA.pai_tracker.member_vars[ 730 "global_best_validation_score" 731 ] 732 733 if GPA.pc.get_verbose(): 734 print( 735 f"In statements to check next learning rate with " 736 f"stepped {stepped} and max count {at_last_count}" 737 ) 738 739 # If no scores saved for this dendrite and initial LR test did second step 740 if len( 741 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] 742 ) == 0 and ( 743 GPA.pai_tracker.member_vars["current_step_count"] 744 - GPA.pai_tracker.member_vars[ 745 "current_n_learning_rate_initial_skip_steps" 746 ] 747 == 2 748 or at_last_count 749 ): 750 751 restructured = True 752 GPA.pai_tracker.clear_optimizer_and_scheduler() 753 754 # Save system for this initial condition 755 old_global = GPA.pai_tracker.member_vars["global_best_validation_score"] 756 old_accuracy = GPA.pai_tracker.member_vars[ 757 "current_best_validation_score" 758 ] 759 old_counts = GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] 760 skip1 = GPA.pai_tracker.member_vars[ 761 "current_n_learning_rate_initial_skip_steps" 762 ] 763 764 now = datetime.now() 765 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 766 767 GPA.pai_tracker.save_graphs( 768 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 769 ) 770 771 if GPA.pc.get_test_saves(): 772 UPA.save_system( 773 net, 774 GPA.pc.get_save_name(), 775 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 776 ) 777 778 if GPA.pc.get_verbose(): 779 print( 780 f"Saving with initial steps: {dt_string}_PBCount_" 781 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 782 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 783 f"with current best {old_accuracy}" 784 ) 785 786 # Load back at start and try with lower initial learning rate 787 net = UPA.load_system( 788 net, 789 GPA.pc.get_save_name(), 790 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 791 switch_call=True, 792 ) 793 GPA.pai_tracker.member_vars[ 794 "current_n_learning_rate_initial_skip_steps" 795 ] = (skip1 + 1) 796 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"].append( 797 old_accuracy 798 ) 799 GPA.pai_tracker.member_vars["global_best_validation_score"] = old_global 800 GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] = old_counts 801 802 # If there is one score already, this is first step at next score 803 elif len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 1: 804 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"].append( 805 GPA.pai_tracker.member_vars["current_best_validation_score"] 806 ) 807 808 # If this LR's score was worse than last LR's score 809 lr_score_worse = False 810 if GPA.pai_tracker.member_vars["maximizing_score"]: 811 lr_score_worse = ( 812 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0] 813 > GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1] 814 ) 815 else: 816 lr_score_worse = ( 817 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0] 818 < GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1] 819 ) 820 821 if lr_score_worse: 822 restructured = True 823 GPA.pai_tracker.clear_optimizer_and_scheduler() 824 825 if GPA.pc.get_verbose(): 826 print( 827 f'Got initial {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1} ' 828 f'step score {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0]} ' 829 f'and {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 830 f'score at step {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1]} ' 831 f"so loading old score" 832 ) 833 834 prior_best = GPA.pai_tracker.member_vars[ 835 "current_cycle_lr_max_scores" 836 ][0] 837 838 now = datetime.now() 839 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 840 841 GPA.pai_tracker.save_graphs( 842 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 843 ) 844 845 if GPA.pc.get_test_saves(): 846 UPA.save_system( 847 net, 848 GPA.pc.get_save_name(), 849 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 850 ) 851 852 if GPA.pc.get_verbose(): 853 print( 854 f"Saving with initial steps: {dt_string}_PBCount_" 855 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 856 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 857 ) 858 859 if GPA.pc.get_test_saves(): 860 net = UPA.load_system( 861 net, 862 GPA.pc.get_save_name(), 863 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1}', 864 switch_call=True, 865 ) 866 867 # Save graphs for chosen one 868 now = datetime.now() 869 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 870 871 GPA.pai_tracker.save_graphs( 872 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}PICKED' 873 ) 874 875 if GPA.pc.get_test_saves(): 876 UPA.save_system( 877 net, 878 GPA.pc.get_save_name(), 879 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 880 ) 881 882 if GPA.pc.get_verbose(): 883 print( 884 f"Saving with initial steps: {dt_string}_PBCount_" 885 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 886 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 887 ) 888 889 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 890 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 891 GPA.pai_tracker.member_vars["current_step_count"] 892 ) 893 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 894 learning_rate2 895 ) 896 GPA.pai_tracker.member_vars["current_best_validation_score"] = ( 897 prior_best 898 ) 899 900 if GPA.pc.get_verbose(): 901 print( 902 f"Setting last max steps to " 903 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 904 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 905 ) 906 907 else: # Current LR score is better 908 if GPA.pc.get_verbose(): 909 print( 910 f'Got initial {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1} ' 911 f'step score {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0]} ' 912 f'and {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 913 f'score at step {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1]} ' 914 f"so NOT loading old score and continuing with this score" 915 ) 916 917 if at_last_count: # If this is the last one, set it to be picked 918 restructured = True 919 GPA.pai_tracker.clear_optimizer_and_scheduler() 920 921 now = datetime.now() 922 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 923 924 GPA.pai_tracker.save_graphs( 925 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}PICKED' 926 ) 927 928 if GPA.pc.get_test_saves(): 929 UPA.save_system( 930 net, 931 GPA.pc.get_save_name(), 932 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 933 ) 934 935 if GPA.pc.get_verbose(): 936 print( 937 f"Saving with initial steps: {dt_string}_PBCount_" 938 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 939 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 940 ) 941 942 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 943 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 944 GPA.pai_tracker.member_vars["current_step_count"] 945 ) 946 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 947 learning_rate2 948 ) 949 950 if GPA.pc.get_verbose(): 951 print( 952 f"Setting last max steps to " 953 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 954 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 955 ) 956 957 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 958 959 elif len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 2: 960 print( 961 "Should never be here. Please let Perforated AI know if this happened." 962 ) 963 pdb.set_trace() 964 965 GPA.pai_tracker.member_vars["global_best_validation_score"] = ( 966 best_score_so_far 967 ) 968 969 else: 970 if GPA.pc.get_verbose(): 971 print( 972 f"Setting last max steps to " 973 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 974 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 975 ) 976 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] += 1 977 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = learning_rate2 978 if restructured: 979 return NETWORK_RESTRUCTURED, net 980 else: 981 return NO_MODEL_UPDATE, net
Updates the scheduler
This increments the scheduler, but if we are automatically sweeping to find the best initial learning rate for a new set of dendrites this function also triggers the network at addition time to try the next value.
Process for finding best initial learning rate for dendrites:
- Start at default rate
- Learn at that rate until scheduler increments twice
- Save that version, start dendrites at LR current increment - 1
- Repeat 2 and 3 until version has worse final score at set LR
- Load previous model with best accuracy at that LR as initial rate
Parameters
- net (torch.nn.Module): The neural network model being trained.
- accuracy (float): The accuracy of the model at the current learning rate.
- epochs_since_cycle_switch (int): The number of epochs since the last cycle switch.
Returns
- int: The status of restructuring or training completion.
- torch.nn.Module: The potentially modified neural network model.
class
PAINeuronModuleTracker:
984class PAINeuronModuleTracker: 985 """ 986 Manager class that tracks all neuron layers and dendrite layers, 987 controls when new dendrites are added, and communicates signals to modules. 988 """ 989 990 def __init__( 991 self, 992 doing_pai, 993 save_name, 994 making_graphs=True, 995 param_vals_setting=-1, 996 values_per_train_epoch=-1, 997 values_per_val_epoch=-1, 998 ): 999 """Initialize the tracker 1000 1001 Parameters 1002 ---------- 1003 doing_pai : bool 1004 Whether or not dendrites should be used. 1005 save_name : str 1006 The base name for saving models and graphs. 1007 making_graphs : bool, optional 1008 Whether or not to generate graphs, by default True. 1009 param_vals_setting : int, optional 1010 Parameter values setting, by default -1. 1011 values_per_train_epoch : int, optional 1012 The number of values to look back for graphing 1013 during training, by default -1 (all values). 1014 values_per_val_epoch : int, optional 1015 The number of values to look back for graphing 1016 during validation, by default -1 (all values). 1017 Returns 1018 ------- 1019 None 1020 """ 1021 1022 # Dict of member vars and their types for saving 1023 self.member_vars = {} 1024 self.member_var_types = {} 1025 1026 # Whether or not PAI will be running 1027 self.member_vars["doing_pai"] = doing_pai 1028 self.member_var_types["doing_pai"] = "bool" 1029 1030 # How many Dendrites have been added 1031 self.member_vars["num_dendrites_added"] = 0 1032 self.member_var_types["num_dendrites_added"] = "int" 1033 1034 # How many Dendrites have been successfully integrated, does not count currently training dendrites 1035 self.member_vars["num_dendrites_integrated"] = 0 1036 self.member_var_types["num_dendrites_integrated"] = "int" 1037 1038 # How many cycles have been run, *2 or *2+1 of the above 1039 self.member_vars["num_cycles"] = 0 1040 self.member_var_types["num_cycles"] = "int" 1041 1042 # Pointers to all neuron wrapped modules 1043 self.neuron_module_vector = [] 1044 1045 # Pointers to all non neuron modules for tracking 1046 self.tracked_neuron_module_vector = [] 1047 1048 # Neuron training or dendrite training mode 1049 self.member_vars["mode"] = "n" 1050 self.member_var_types["mode"] = "string" 1051 1052 # Number of epochs run excluding overwritten epochs 1053 self.member_vars["num_epochs_run"] = -1 1054 self.member_var_types["num_epochs_run"] = "int" 1055 1056 # Number including overwritten epochs 1057 self.member_vars["total_epochs_run"] = -1 1058 self.member_var_types["total_epochs_run"] = "int" 1059 1060 # Last epoch that validation/correlation score was improved 1061 self.member_vars["epoch_last_improved"] = 0 1062 self.member_var_types["epoch_last_improved"] = "int" 1063 1064 # Running validation accuracy 1065 self.member_vars["running_accuracy"] = 0 1066 self.member_var_types["running_accuracy"] = "float" 1067 1068 # True if maxing validation, False if minimizing Loss 1069 self.member_vars["maximizing_score"] = True 1070 self.member_var_types["maximizing_score"] = "bool" 1071 1072 # Mode for switching back and forth between learning modes 1073 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 1074 self.member_var_types["switch_mode"] = "int" 1075 1076 # Epoch of the last switch 1077 self.member_vars["last_switch"] = 0 1078 self.member_var_types["last_switch"] = "int" 1079 1080 # Highest validation score from current cycle 1081 self.member_vars["current_best_validation_score"] = 0 1082 self.member_var_types["current_best_validation_score"] = "float" 1083 1084 # Last epoch where the learning rate was updated 1085 self.member_vars["initial_lr_test_epoch_count"] = -1 1086 self.member_var_types["initial_lr_test_epoch_count"] = "int" 1087 1088 # Highest validation score of full run 1089 self.member_vars["global_best_validation_score"] = 0 1090 self.member_var_types["global_best_validation_score"] = "float" 1091 1092 # List of switch epochs 1093 self.member_vars["switch_epochs"] = [] 1094 self.member_var_types["switch_epochs"] = "int array" 1095 1096 # Parameter counts at each network structure 1097 self.member_vars["param_counts"] = [] 1098 self.member_var_types["param_counts"] = "int array" 1099 1100 # List of epochs where switch was made to neuron training 1101 self.member_vars["n_switch_epochs"] = [] 1102 self.member_var_types["n_switch_epochs"] = "int array" 1103 1104 # List of epochs where switch was made to dendrite training 1105 self.member_vars["p_switch_epochs"] = [] 1106 self.member_var_types["p_switch_epochs"] = "int array" 1107 1108 # List of validation accuracies 1109 self.member_vars["accuracies"] = [] 1110 self.member_var_types["accuracies"] = "float array" 1111 1112 # List of epochs where score improved for scheduler updates 1113 self.member_vars["last_improved_accuracies"] = [] 1114 self.member_var_types["last_improved_accuracies"] = "int array" 1115 1116 # List of test accuracy scores registered 1117 self.member_vars["test_accuracies"] = [] 1118 self.member_var_types["test_accuracies"] = "float array" 1119 1120 # List of accuracies registered during neuron training 1121 self.member_vars["n_accuracies"] = [] 1122 self.member_var_types["n_accuracies"] = "float array" 1123 1124 # List of accuracies registered during dendrite training 1125 self.member_vars["p_accuracies"] = [] 1126 self.member_var_types["p_accuracies"] = "float array" 1127 1128 # Running average accuracies from recent epochs 1129 self.member_vars["running_accuracies"] = [] 1130 self.member_var_types["running_accuracies"] = "float array" 1131 1132 # List of additional scores recorded 1133 self.member_vars["extra_scores"] = {} 1134 self.member_var_types["extra_scores"] = "float array dictionary" 1135 1136 # Extra scores not set to be graphed 1137 self.member_vars["extra_scores_without_graphing"] = {} 1138 self.member_var_types["extra_scores_without_graphing"] = ( 1139 "float array dictionary" 1140 ) 1141 1142 # List of test scores 1143 self.member_vars["test_scores"] = [] 1144 self.member_var_types["test_scores"] = "float array" 1145 1146 # Extra scores calculated during neuron training 1147 self.member_vars["n_extra_scores"] = {} 1148 self.member_var_types["n_extra_scores"] = "float array dictionary" 1149 1150 # List of training losses calculated 1151 self.member_vars["training_loss"] = [] 1152 self.member_var_types["training_loss"] = "float array" 1153 1154 # List of learning rates at each epoch 1155 self.member_vars["training_learning_rates"] = [] 1156 self.member_var_types["training_learning_rates"] = "float array" 1157 1158 # Best dendrite scores 1159 self.member_vars["best_scores"] = [] 1160 self.member_var_types["best_scores"] = "float array array" 1161 1162 # Current dendrite scores 1163 self.member_vars["current_scores"] = [] 1164 self.member_var_types["current_scores"] = "float array array" 1165 1166 # Times for neuron training epochs 1167 self.member_vars["n_epoch_times"] = [] 1168 self.member_var_types["n_epoch_times"] = "float array" 1169 1170 # Timing values 1171 self.member_vars["p_epoch_times"] = [] 1172 self.member_var_types["p_epoch_times"] = "float array" 1173 self.member_vars["n_train_times"] = [] 1174 self.member_var_types["n_train_times"] = "float array" 1175 self.member_vars["p_train_times"] = [] 1176 self.member_var_types["p_train_times"] = "float array" 1177 self.member_vars["n_val_times"] = [] 1178 self.member_var_types["n_val_times"] = "float array" 1179 self.member_vars["p_val_times"] = [] 1180 self.member_var_types["p_val_times"] = "float array" 1181 1182 # Setting for tracking timing 1183 self.member_vars["manual_train_switch"] = False 1184 self.member_var_types["manual_train_switch"] = "bool" 1185 1186 # Tracking scores overwritten when reloading best model 1187 self.member_vars["overwritten_extras"] = [] 1188 self.member_var_types["overwritten_extras"] = "float array dictionary array" 1189 self.member_vars["overwritten_vals"] = [] 1190 self.member_var_types["overwritten_vals"] = "float array array" 1191 self.member_vars["overwritten_epochs"] = 0 1192 self.member_var_types["overwritten_epochs"] = "int" 1193 1194 # Setting for determining scores 1195 self.member_vars["param_vals_setting"] = GPA.pc.get_param_vals_setting() 1196 self.member_var_types["param_vals_setting"] = "int" 1197 1198 # Optimizer and scheduler types and instances 1199 self.member_vars["optimizer"] = None 1200 self.member_var_types["optimizer"] = "type" 1201 self.member_vars["scheduler"] = None 1202 self.member_var_types["scheduler"] = "type" 1203 self.member_vars["optimizer_instance"] = None 1204 self.member_var_types["optimizer_instance"] = "empty array" 1205 self.member_vars["scheduler_instance"] = None 1206 self.member_var_types["scheduler_instance"] = "empty array" 1207 1208 # Flag for if the tracker was loaded 1209 self.loaded = False 1210 1211 # flag for 1212 self.member_vars["step_status"] = STEP_CLEARED 1213 self.member_var_types["step_status"] = "int" 1214 1215 1216 # Settings for tracking learning rates 1217 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 1218 self.member_var_types["current_n_learning_rate_initial_skip_steps"] = "int" 1219 self.member_vars["last_max_learning_rate_steps"] = 0 1220 self.member_var_types["last_max_learning_rate_steps"] = "int" 1221 self.member_vars["last_max_learning_rate_value"] = -1 1222 self.member_var_types["last_max_learning_rate_value"] = "float" 1223 self.member_vars["current_cycle_lr_max_scores"] = [] 1224 self.member_var_types["current_cycle_lr_max_scores"] = "float array" 1225 self.member_vars["current_step_count"] = 0 1226 self.member_var_types["current_step_count"] = "int" 1227 self.member_vars["committed_to_initial_rate"] = True 1228 self.member_var_types["committed_to_initial_rate"] = "bool" 1229 self.member_vars["best_mean_score_improved_this_epoch"] = 0 1230 self.member_var_types["best_mean_score_improved_this_epoch"] = "int" 1231 1232 # Flag for if current dendrite achieved highest global score 1233 self.member_vars["current_n_set_global_best"] = True 1234 self.member_var_types["current_n_set_global_best"] = "bool" 1235 1236 # Number of tries adding this dendrite count 1237 self.member_vars["num_dendrite_tries"] = 0 1238 self.member_var_types["num_dendrite_tries"] = "int" 1239 1240 # Count of batches per epoch 1241 self.values_per_train_epoch = values_per_train_epoch 1242 self.values_per_val_epoch = values_per_val_epoch 1243 1244 self.save_name = save_name 1245 self.making_graphs = making_graphs 1246 1247 self.start_time = time.time() 1248 self.saved_time = 0 1249 self.start_epoch(internal_call=True) 1250 1251 if GPA.pc.get_verbose(): 1252 print(f'Initializing with switch_mode {self.member_vars["switch_mode"]}') 1253 1254 def to_string(self): 1255 """Convert tracker values to string for saving with safetensors. 1256 1257 Parameters 1258 ---------- 1259 None 1260 1261 Returns 1262 ------- 1263 str 1264 Serialized tracker state suitable for storage in a safetensors field. 1265 """ 1266 1267 full_string = "" 1268 for var in self.member_vars: 1269 full_string += var + "," 1270 if self.member_vars[var] is None: 1271 full_string += "None" 1272 full_string += "\n" 1273 elif self.member_var_types[var] == "bool": 1274 full_string += str(self.member_vars[var]) 1275 full_string += "\n" 1276 elif self.member_var_types[var] in ("int", "float", "string"): 1277 full_string += str(self.member_vars[var]) 1278 full_string += "\n" 1279 elif self.member_var_types[var] == "type": 1280 name = ( 1281 self.member_vars[var].__module__ 1282 + "." 1283 + self.member_vars[var].__name__ 1284 ) 1285 full_string += str(self.member_vars[var]) 1286 full_string += "\n" 1287 elif self.member_var_types[var] == "empty array": 1288 full_string += "[]" 1289 full_string += "\n" 1290 elif self.member_var_types[var] in ("int array", "float array"): 1291 full_string += "\n" 1292 string = "" 1293 for val in self.member_vars[var]: 1294 string += str(val) + "," 1295 # Remove the last comma 1296 string = string[:-1] 1297 full_string += string 1298 full_string += "\n" 1299 elif self.member_var_types[var] == "float array dictionary array": 1300 full_string += "\n" 1301 for array in self.member_vars[var]: 1302 for key in array: 1303 string = key + "," 1304 for val in array[key]: 1305 string += str(val) + "," 1306 # Remove the last comma 1307 string = string[:-1] 1308 full_string += string 1309 full_string += "\n" 1310 full_string += "endkey" 1311 full_string += "\n" 1312 full_string += "endarray" 1313 full_string += "\n" 1314 elif self.member_var_types[var] == "float array dictionary": 1315 full_string += "\n" 1316 for key in self.member_vars[var]: 1317 string = key + "," 1318 for val in self.member_vars[var][key]: 1319 string += str(val) + "," 1320 # Remove the last comma 1321 string = string[:-1] 1322 full_string += string 1323 full_string += "\n" 1324 full_string += "end" 1325 full_string += "\n" 1326 elif self.member_var_types[var] == "float array array": 1327 full_string += "\n" 1328 for array in self.member_vars[var]: 1329 string = "" 1330 for val in array: 1331 string += str(val) + "," 1332 # Remove the last comma 1333 string = string[:-1] 1334 full_string += string 1335 full_string += "\n" 1336 full_string += "end" 1337 full_string += "\n" 1338 else: 1339 print("Did not find a member variable") 1340 pdb.set_trace() 1341 return full_string 1342 1343 def from_string(self, string): 1344 """Load tracker values from string. 1345 1346 Parameters 1347 ---------- 1348 string : str 1349 The string to load from. 1350 1351 Returns 1352 ------- 1353 None 1354 This function does not return a value. 1355 """ 1356 f = io.StringIO(string) 1357 while True: 1358 line = f.readline() 1359 if not line: 1360 break 1361 vals = line.split(",") 1362 var = vals[0] 1363 1364 if self.member_var_types[var] == "bool": 1365 val = vals[1][:-1] 1366 if val == "True": 1367 self.member_vars[var] = True 1368 elif val == "False": 1369 self.member_vars[var] = False 1370 elif val == "1": 1371 self.member_vars[var] = 1 1372 elif val == "0": 1373 self.member_vars[var] = 0 1374 else: 1375 print("Something went wrong with loading") 1376 pdb.set_trace() 1377 elif self.member_var_types[var] == "int": 1378 val = vals[1] 1379 self.member_vars[var] = int(val) 1380 elif self.member_var_types[var] == "float": 1381 val = vals[1] 1382 self.member_vars[var] = float(val) 1383 elif self.member_var_types[var] == "string": 1384 val = vals[1][:-1] 1385 self.member_vars[var] = val 1386 elif self.member_var_types[var] == "type": 1387 # Ignore loading types, tracker should have them set up 1388 continue 1389 elif self.member_var_types[var] == "empty array": 1390 val = vals[1] 1391 self.member_vars[var] = [] 1392 elif self.member_var_types[var] == "int array": 1393 vals = f.readline()[:-1].split(",") 1394 self.member_vars[var] = [] 1395 if vals[0] == "": 1396 continue 1397 for val in vals: 1398 self.member_vars[var].append(int(val)) 1399 elif self.member_var_types[var] == "float array": 1400 vals = f.readline()[:-1].split(",") 1401 self.member_vars[var] = [] 1402 if vals[0] == "": 1403 continue 1404 for val in vals: 1405 self.member_vars[var].append(float(val)) 1406 elif self.member_var_types[var] == "float array dictionary array": 1407 self.member_vars[var] = [] 1408 line2 = f.readline()[:-1] 1409 while line2 != "endarray": 1410 temp = {} 1411 while line2 != "endkey": 1412 vals = line2.split(",") 1413 name = vals[0] 1414 temp[name] = [] 1415 vals = vals[1:] 1416 for val in vals: 1417 temp[name].append(float(val)) 1418 line2 = f.readline()[:-1] 1419 self.member_vars[var].append(temp) 1420 line2 = f.readline()[:-1] 1421 elif self.member_var_types[var] == "float array dictionary": 1422 self.member_vars[var] = {} 1423 line2 = f.readline()[:-1] 1424 while line2 != "end": 1425 vals = line2.split(",") 1426 name = vals[0] 1427 self.member_vars[var][name] = [] 1428 vals = vals[1:] 1429 for val in vals: 1430 self.member_vars[var][name].append(float(val)) 1431 line2 = f.readline()[:-1] 1432 elif self.member_var_types[var] == "float array array": 1433 self.member_vars[var] = [] 1434 line2 = f.readline()[:-1] 1435 while line2 != "end": 1436 vals = line2.split(",") 1437 self.member_vars[var].append([]) 1438 if line2: 1439 for val in vals: 1440 self.member_vars[var][-1].append(float(val)) 1441 line2 = f.readline()[:-1] 1442 else: 1443 print("Did not find a member variable") 1444 1445 pdb.set_trace() 1446 1447 def from_string_debug(self, string): 1448 """Debug function to print tracker values from string without loading them. 1449 1450 Parameters 1451 ---------- 1452 string : str 1453 The string to debug load from. 1454 1455 Returns 1456 ------- 1457 None 1458 This function does not return a value. 1459 """ 1460 f = io.StringIO(string) 1461 print("=== DEBUGGING TRACKER VARIABLES ===") 1462 1463 while True: 1464 line = f.readline() 1465 if not line: 1466 break 1467 vals = line.split(",") 1468 var = vals[0] 1469 1470 print(f"\nVariable: {var}") 1471 print(f"Type: {self.member_var_types.get(var, 'UNKNOWN TYPE')}") 1472 print(f"Current value: {self.member_vars.get(var, 'NOT SET')}") 1473 1474 if self.member_var_types.get(var) == "bool": 1475 val = vals[1][:-1] 1476 print(f"Would set to: {val} -> {val == 'True'}") 1477 1478 elif self.member_var_types.get(var) == "int": 1479 val = vals[1] 1480 print(f"Would set to: {int(val)}") 1481 1482 elif self.member_var_types.get(var) == "float": 1483 val = vals[1] 1484 print(f"Would set to: {float(val)}") 1485 1486 elif self.member_var_types.get(var) == "string": 1487 val = vals[1][:-1] 1488 print(f"Would set to: '{val}'") 1489 1490 elif self.member_var_types.get(var) == "type": 1491 print("Would skip (type loading)") 1492 1493 elif self.member_var_types.get(var) == "empty array": 1494 val = vals[1] 1495 print(f"Would set to: [] (empty array)") 1496 1497 elif self.member_var_types.get(var) == "int array": 1498 vals_line = f.readline()[:-1].split(",") 1499 print(f"Would set to int array with {len(vals_line)} elements:") 1500 if vals_line[0] != "": 1501 print( 1502 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1503 ) 1504 else: 1505 print(" Empty array") 1506 1507 elif self.member_var_types.get(var) == "float array": 1508 vals_line = f.readline()[:-1].split(",") 1509 print(f"Would set to float array with {len(vals_line)} elements:") 1510 if vals_line[0] != "": 1511 print( 1512 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1513 ) 1514 else: 1515 print(" Empty array") 1516 1517 elif self.member_var_types.get(var) == "float array dictionary array": 1518 print("Would process float array dictionary array:") 1519 array_count = 0 1520 line2 = f.readline()[:-1] 1521 while line2 != "endarray": 1522 key_count = 0 1523 while line2 != "endkey": 1524 vals_dict = line2.split(",") 1525 name = vals_dict[0] 1526 print( 1527 f" Array {array_count}, Key '{name}': {len(vals_dict)-1} elements" 1528 ) 1529 key_count += 1 1530 line2 = f.readline()[:-1] 1531 print(f" Array {array_count} has {key_count} keys") 1532 array_count += 1 1533 line2 = f.readline()[:-1] 1534 print(f" Total arrays: {array_count}") 1535 1536 elif self.member_var_types.get(var) == "float array dictionary": 1537 print("Would process float array dictionary:") 1538 line2 = f.readline()[:-1] 1539 key_count = 0 1540 while line2 != "end": 1541 vals_dict = line2.split(",") 1542 name = vals_dict[0] 1543 print(f" Key '{name}': {len(vals_dict)-1} elements") 1544 key_count += 1 1545 line2 = f.readline()[:-1] 1546 print(f" Total keys: {key_count}") 1547 1548 elif self.member_var_types.get(var) == "float array array": 1549 print("Would process float array array:") 1550 line2 = f.readline()[:-1] 1551 array_count = 0 1552 while line2 != "end": 1553 if line2: 1554 vals_array = line2.split(",") 1555 print(f" Array {array_count}: {len(vals_array)} elements") 1556 else: 1557 print(f" Array {array_count}: empty") 1558 array_count += 1 1559 line2 = f.readline()[:-1] 1560 print(f" Total arrays: {array_count}") 1561 1562 else: 1563 print(f"UNKNOWN TYPE: {self.member_var_types.get(var, 'NOT FOUND')}") 1564 1565 print("\n=== END DEBUG ===") 1566 1567 def save_tracker_settings(self): 1568 """Save tracker settings for DistributedDataParallel use. 1569 1570 Saves settings in save_name/array_dims.csv 1571 1572 Parameters 1573 ---------- 1574 None 1575 Returns 1576 ------- 1577 None 1578 1579 ----- 1580 Instructions for use are in API customization.md 1581 """ 1582 if not os.path.isdir(self.save_name): 1583 os.makedirs(self.save_name) 1584 f = open(self.save_name + "/array_dims.csv", "w") 1585 for layer in self.neuron_module_vector: 1586 dv = layer.dendrite_module.dendrite_values[0] 1587 shape_str = ",".join(str(s) for s in dv.dendrite_storage_shape.tolist()) 1588 f.write(f"{layer.name},{shape_str}\n") 1589 f.close() 1590 if not GPA.pc.get_silent(): 1591 print("Tracker settings saved.") 1592 print("You may now delete save_tracker_settings") 1593 1594 def initialize_tracker_settings(self): 1595 """Initialize tracker settings from saved file. 1596 1597 This function loads tracker settings from a CSV file and applies them 1598 to the layers the tracker is managing. 1599 1600 Parameters 1601 ---------- 1602 None 1603 1604 Returns 1605 ------- 1606 None 1607 1608 """ 1609 1610 channels = {} 1611 if not os.path.exists(self.save_name + "/array_dims.csv"): 1612 print( 1613 "You must call save_tracker_settings before " 1614 "initialize_tracker_settings" 1615 ) 1616 print("Follow instructions in customization.md") 1617 pdb.set_trace() 1618 f = open(self.save_name + "/array_dims.csv", "r") 1619 for line in f: 1620 parts = line.strip().split(",") 1621 channels[parts[0]] = [int(s) for s in parts[1:]] 1622 for layer in self.neuron_module_vector: 1623 dv = layer.dendrite_module.dendrite_values[0] 1624 dv.setup_arrays(channels[layer.name]) 1625 1626 def set_optimizer_instance(self, optimizer_instance, additional_optimizers=[]): 1627 """Set optimizer instance directly. 1628 1629 Parameters 1630 ---------- 1631 optimizer_instance : object 1632 The optimizer instance to set. 1633 1634 Returns 1635 ------- 1636 None 1637 1638 """ 1639 # This call must be first before the parameters get filtered. 1640 optimizer_instance.zero_grad() 1641 try: 1642 for param_group in optimizer_instance.param_groups: 1643 if ( 1644 param_group["weight_decay"] > 0 1645 and GPA.pc.get_weight_decay_accepted() is False 1646 ): 1647 _pai_log( 1648 "warning", 1649 "For PAI training it is recommended to not use weight decay in your optimizer", 1650 ) 1651 1652 except: 1653 pass 1654 self.member_vars["optimizer_instance"] = optimizer_instance 1655 if GPA.pc.get_perforated_backpropagation(): 1656 TPB.setup_optimizer_pb(self.member_vars["optimizer_instance"]) 1657 for optimizer in additional_optimizers: 1658 TPB.filter_params(optimizer) 1659 1660 def set_optimizer(self, optimizer): 1661 """Set optimizer type to be initialized later 1662 1663 Parameters 1664 ---------- 1665 optimizer : object 1666 The optimizer type to set. 1667 1668 Returns 1669 ------- 1670 None 1671 1672 """ 1673 self.member_vars["optimizer"] = optimizer 1674 1675 def set_scheduler(self, scheduler): 1676 """Set scheduler type to be initialized later 1677 1678 Parameters 1679 ---------- 1680 scheduler : object 1681 The scheduler type to set. 1682 1683 Returns 1684 ------- 1685 None 1686 1687 """ 1688 if scheduler is not torch.optim.lr_scheduler.ReduceLROnPlateau: 1689 if GPA.pc.get_verbose(): 1690 print("Not using ReduceLROnPlateau, this is not recommended") 1691 self.member_vars["scheduler"] = scheduler 1692 1693 def increment_scheduler(self, num_ticks, mode): 1694 """Increment the scheduler a set number of times. 1695 1696 Used for finding best initial learning rate when adding dendrites. 1697 1698 Parameters 1699 ---------- 1700 num_ticks : int 1701 The number of scheduler steps to take. 1702 mode : str 1703 The mode for stepping the scheduler. Options are: 1704 - "step_learning_rate": Step based on improved accuracy epochs 1705 - "increment_epoch_count": Step based on total epoch count 1706 1707 Returns 1708 ------- 1709 current_steps : int 1710 The number of learning rate changes that occurred. 1711 learning_rate1 : float 1712 The final learning rate after stepping. 1713 1714 """ 1715 1716 current_steps = 0 1717 current_ticker = 0 1718 1719 for param_group in GPA.pai_tracker.member_vars[ 1720 "optimizer_instance" 1721 ].param_groups: 1722 learning_rate1 = param_group["lr"] 1723 1724 if GPA.pc.get_verbose(): 1725 print("Using scheduler:") 1726 print(type(self.member_vars["scheduler_instance"])) 1727 1728 while current_ticker < num_ticks: 1729 if GPA.pc.get_verbose(): 1730 print( 1731 f"Lower start rate initial {learning_rate1} " 1732 f'stepping {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} times' 1733 ) 1734 1735 if ( 1736 type(self.member_vars["scheduler_instance"]) 1737 is torch.optim.lr_scheduler.ReduceLROnPlateau 1738 ): 1739 if mode == "step_learning_rate": 1740 # Step with counter as last improved accuracy 1741 self.member_vars["scheduler_instance"].step( 1742 metrics=self.member_vars["last_improved_accuracies"][ 1743 GPA.pai_tracker.steps_after_switch() - 1 1744 ] 1745 ) 1746 elif mode == "increment_epoch_count": 1747 # Step with improved epoch counts up to current location 1748 self.member_vars["scheduler_instance"].step( 1749 metrics=self.member_vars["last_improved_accuracies"][ 1750 -((num_ticks - 1) - current_ticker) - 1 1751 ] 1752 ) 1753 else: 1754 self.member_vars["scheduler_instance"].step() 1755 1756 for param_group in GPA.pai_tracker.member_vars[ 1757 "optimizer_instance" 1758 ].param_groups: 1759 learning_rate2 = param_group["lr"] 1760 1761 if learning_rate2 != learning_rate1: 1762 current_steps += 1 1763 learning_rate1 = learning_rate2 1764 if mode == "step_learning_rate": 1765 current_ticker += 1 1766 if GPA.pc.get_verbose(): 1767 print(f"1 step {current_steps} to {learning_rate2}") 1768 1769 if mode == "increment_epoch_count": 1770 current_ticker += 1 1771 1772 return current_steps, learning_rate1 1773 1774 def setup_optimizer(self, net, opt_args, sched_args=None, parameters=None): 1775 """Initialize the optimizer and scheduler when added. 1776 1777 Parameters 1778 ---------- 1779 net : object 1780 The neural network model. 1781 opt_args : dict 1782 The arguments for the optimizer. 1783 sched_args : dict, optional 1784 The arguments for the scheduler, by default None. 1785 1786 Returns 1787 ------- 1788 optimizer : object 1789 The initialized optimizer instance. 1790 scheduler : object or None 1791 The initialized scheduler instance, or None if no scheduler was set. 1792 1793 """ 1794 if "weight_decay" in opt_args and not GPA.pc.get_weight_decay_accepted(): 1795 _pai_log( 1796 "warning", 1797 "For PAI training it is recommended to not use weight decay in your optimizer", 1798 ) 1799 1800 if ("model" not in opt_args.keys()) and "params" not in opt_args.keys(): 1801 print("In setup_optimizer it will be depreciated to not pass in params yourself in the future") 1802 print("please change the settings to include params") 1803 if self.member_vars["mode"] == "n": 1804 if parameters is not None: 1805 opt_args["params"] = parameters 1806 else: 1807 opt_args["params"] = filter(lambda p: p.requires_grad, net.parameters()) 1808 else: 1809 params = UPA.get_pai_network_params(net) 1810 if parameters is not None: 1811 # Filter parameters to only those in params, preserving weight_decay 1812 params_set = set(params) 1813 filtered_params = [] 1814 for param_group in parameters: 1815 filtered_group_params = [p for p in param_group["params"] if p in params_set] 1816 if filtered_group_params: 1817 filtered_params.append({ 1818 "params": filtered_group_params, 1819 "weight_decay": param_group["weight_decay"] 1820 }) 1821 opt_args["params"] = filtered_params 1822 else: 1823 opt_args["params"] = params 1824 elif "params" in opt_args.keys(): 1825 # Check if params is a list of param groups (dicts) or a single param group 1826 params_value = opt_args["params"] 1827 if isinstance(params_value, list) and len(params_value) > 0: 1828 # Check if it's a list of dicts (multiple param groups) or list of tensors (single group) 1829 if isinstance(params_value[0], dict): 1830 # Multiple param groups format: [{"params": [...], "lr": ...}, ...] 1831 # Filter each param group for requires_grad 1832 filtered_param_groups = [] 1833 for param_group in params_value: 1834 filtered_group_params = [p for p in param_group["params"] if p.requires_grad] 1835 if filtered_group_params: 1836 new_group = param_group.copy() 1837 new_group["params"] = filtered_group_params 1838 filtered_param_groups.append(new_group) 1839 opt_args["params"] = filtered_param_groups 1840 else: 1841 # Single param group format: [tensor1, tensor2, ...] or generator 1842 # Filter for requires_grad 1843 opt_args["params"] = [p for p in params_value if p.requires_grad] 1844 elif hasattr(params_value, '__iter__'): 1845 # Handle generators or other iterables 1846 opt_args["params"] = [p for p in params_value if p.requires_grad] 1847 1848 optimizer = self.member_vars["optimizer"](**opt_args) 1849 self.set_optimizer_instance(optimizer) 1850 1851 if self.member_vars["scheduler"] is not None: 1852 # Handle SequentialLR specially 1853 if self.member_vars["scheduler"] is torch.optim.lr_scheduler.SequentialLR: 1854 """ 1855 sched_args should be a dict with "schedulers" (list of tuples) and "milestones" 1856 For example: 1857 sequential_schedArgs = { 1858 "schedulers": [ 1859 (warmup_scheduler_class, warmup_schedArgs), 1860 (main_scheduler_class, main_schedArgs) 1861 ], 1862 "milestones": [switch_epoch] 1863 } 1864 """ 1865 schedulers = [] 1866 milestones = sched_args.get("milestones", []) 1867 scheduler_configs = sched_args.get("schedulers", []) 1868 1869 for scheduler_class, scheduler_args in scheduler_configs: 1870 schedulers.append(scheduler_class(optimizer, **scheduler_args)) 1871 1872 self.member_vars["scheduler_instance"] = torch.optim.lr_scheduler.SequentialLR( 1873 optimizer, schedulers=schedulers, milestones=milestones 1874 ) 1875 else: 1876 self.member_vars["scheduler_instance"] = self.member_vars["scheduler"]( 1877 optimizer, **sched_args 1878 ) 1879 current_steps = 0 1880 1881 for param_group in GPA.pai_tracker.member_vars[ 1882 "optimizer_instance" 1883 ].param_groups: 1884 learning_rate1 = param_group["lr"] 1885 1886 if GPA.pc.get_verbose(): 1887 print( 1888 f"Resetting scheduler with {GPA.pai_tracker.steps_after_switch()} " 1889 f"steps and {GPA.pc.get_initial_history_after_switches()} initial ticks to skip" 1890 ) 1891 1892 # Find setting of previously used learning rate before adding dendrites 1893 if ( 1894 GPA.pai_tracker.member_vars[ 1895 "current_n_learning_rate_initial_skip_steps" 1896 ] 1897 != 0 1898 ): 1899 additional_steps, learning_rate1 = self.increment_scheduler( 1900 GPA.pai_tracker.member_vars[ 1901 "current_n_learning_rate_initial_skip_steps" 1902 ], 1903 "step_learning_rate", 1904 ) 1905 current_steps += additional_steps 1906 1907 if self.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 1908 initial = GPA.pc.get_initial_history_after_switches() 1909 else: 1910 initial = 0 1911 1912 if GPA.pai_tracker.steps_after_switch() > initial: 1913 # Minus extra 1 because this gets called after start epoch 1914 additional_steps, learning_rate1 = self.increment_scheduler( 1915 (GPA.pai_tracker.steps_after_switch() - initial) - 1, 1916 "increment_epoch_count", 1917 ) 1918 current_steps += additional_steps 1919 1920 if GPA.pc.get_verbose(): 1921 print( 1922 f"Scheduler update loop with {current_steps} " 1923 f"ended with {learning_rate1}" 1924 ) 1925 print( 1926 f"Scheduler ended with {current_steps} steps " 1927 f"and lr of {learning_rate1}" 1928 ) 1929 1930 self.member_vars["current_step_count"] = current_steps 1931 return optimizer, self.member_vars["scheduler_instance"] 1932 else: 1933 return optimizer, None 1934 1935 def clear_optimizer_and_scheduler(self): 1936 """Clear the instances for saving. 1937 1938 Parameters 1939 ---------- 1940 None 1941 1942 Returns 1943 ------- 1944 None 1945 This function does not return a value. 1946 """ 1947 self.member_vars["optimizer_instance"] = None 1948 self.member_vars["scheduler_instance"] = None 1949 1950 def switch_time(self): 1951 """Determine if it's time to switch between neuron and dendrite training. 1952 1953 Parameters 1954 ---------- 1955 None 1956 1957 Returns 1958 ------- 1959 bool 1960 True if it's time to switch, False otherwise. 1961 1962 Notes 1963 ----- 1964 Based on current settings and history of scores. 1965 """ 1966 1967 switch_phrase = "No mode, this should never be the case." 1968 switch_number = GPA.pc.get_n_epochs_to_switch() 1969 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1970 switch_phrase = "DOING_SWITCH_EVERY_TIME" 1971 elif self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY: 1972 switch_phrase = "DOING_HISTORY" 1973 elif self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH: 1974 switch_phrase = "DOING_FIXED_SWITCH" 1975 switch_number = GPA.pc.get_fixed_switch_num() 1976 elif self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1977 switch_phrase = "DOING_NO_SWITCH" 1978 else: 1979 print( 1980 "A switch mode must be set. Check your settings for GPA.pc.set_switch_mode()." 1981 ) 1982 pdb.set_trace() 1983 if not GPA.pc.get_silent(): 1984 if(GPA.pc.get_perforated_backpropagation()): 1985 print( 1986 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1987 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1988 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1989 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1990 f'n: {switch_number}, p: {GPA.pc.get_p_epochs_to_switch()}, ' 1991 f'num_cycles: {self.member_vars["num_cycles"]}' 1992 ) 1993 else: 1994 print( 1995 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1996 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1997 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1998 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1999 f'n: {switch_number}, num_cycles: {self.member_vars["num_cycles"]}' 2000 ) 2001 print( 2002 f' Score tracking: current_n_set_global_best={self.member_vars["current_n_set_global_best"]}, ' 2003 f'global_best={self.member_vars["global_best_validation_score"]:.4f}, ' 2004 f'current_best={self.member_vars["current_best_validation_score"]:.4f}' 2005 ) 2006 if GPA.pc.get_perforated_backpropagation(): 2007 # this will fill in epoch last improved 2008 TPB.best_pai_score_improved_this_epoch(self) ## CLOSED ONLY 2009 if self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 2010 if not GPA.pc.get_silent(): 2011 print("Returning False - doing no switch mode") 2012 return False 2013 2014 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 2015 if not GPA.pc.get_silent(): 2016 print("Returning True - switching every time") 2017 return True 2018 2019 # Check if we're in the middle of learning rate optimization 2020 # If so, block ALL switch triggers until committed 2021 if GPA.pc.get_verbose(): 2022 print("=== LR Optimization Check ===") 2023 print(f' mode == "n": {self.member_vars["mode"] == "n"}') 2024 print(f" get_learn_dendrites_live(): {GPA.pc.get_learn_dendrites_live()}") 2025 print(f' committed_to_initial_rate: {GPA.pai_tracker.member_vars["committed_to_initial_rate"]}') 2026 print(f" get_dont_give_up_unless_learning_rate_lowered(): {GPA.pc.get_dont_give_up_unless_learning_rate_lowered()}") 2027 print(f' current_n_learning_rate_initial_skip_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"]}') 2028 print(f' last_max_learning_rate_steps: {self.member_vars["last_max_learning_rate_steps"]}') 2029 print(f' skip_steps < max_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"] < self.member_vars["last_max_learning_rate_steps"]}') 2030 print(f' scheduler is not None: {self.member_vars["scheduler"] is not None}') 2031 print("=============================") 2032 2033 if ( 2034 ((self.member_vars["mode"] == "n") or GPA.pc.get_learn_dendrites_live()) 2035 and (GPA.pai_tracker.member_vars["committed_to_initial_rate"] is False) 2036 and (GPA.pc.get_dont_give_up_unless_learning_rate_lowered()) 2037 and ( 2038 self.member_vars["current_n_learning_rate_initial_skip_steps"] 2039 <= self.member_vars["last_max_learning_rate_steps"] 2040 ) 2041 and self.member_vars["scheduler"] is not None 2042 ): 2043 if not GPA.pc.get_silent(): 2044 print( 2045 f"Returning False - learning rate optimization in progress. " 2046 f"Not committed yet. Comparing " 2047 f'initial {self.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 2048 f'to last max {self.member_vars["last_max_learning_rate_steps"]}' 2049 ) 2050 return False 2051 2052 if len(self.member_vars["switch_epochs"]) == 0: 2053 this_count = self.member_vars["num_epochs_run"] 2054 else: 2055 this_count = ( 2056 self.member_vars["num_epochs_run"] 2057 - self.member_vars["switch_epochs"][-1] 2058 ) 2059 cap_switch = False 2060 if GPA.pc.get_perforated_backpropagation(): 2061 cap_switch = TPB.check_cap_switch(self, this_count) 2062 2063 if self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY and ( 2064 ( 2065 (self.member_vars["mode"] == "n") 2066 and ( 2067 self.member_vars["num_epochs_run"] 2068 - self.member_vars["epoch_last_improved"] 2069 >= GPA.pc.get_n_epochs_to_switch() 2070 ) 2071 and this_count 2072 >= GPA.pc.get_initial_history_after_switches() 2073 + GPA.pc.get_n_epochs_to_switch() 2074 ) 2075 or (GPA.pc.get_perforated_backpropagation() and TPB.history_switch(self)) 2076 or cap_switch 2077 ): 2078 if not GPA.pc.get_silent(): 2079 print("Returning True - History and last improved is hit") 2080 return True 2081 2082 if self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH and ( 2083 ( 2084 self.member_vars["total_epochs_run"] % GPA.pc.get_fixed_switch_num() 2085 == GPA.pc.get_fixed_switch_num() - 1 2086 ) 2087 and self.member_vars["num_epochs_run"] 2088 >= GPA.pc.get_first_fixed_switch_num() - 1 2089 ): 2090 if not GPA.pc.get_silent(): 2091 print("Returning True - Fixed switch number is hit") 2092 return True 2093 2094 if not GPA.pc.get_silent(): 2095 print("Returning False - no triggers to switch have been hit") 2096 return False 2097 2098 def steps_after_switch(self): 2099 """Based on settings, return value for steps since a switch. 2100 2101 Different options for param vals setting determine what is returned. 2102 2103 Parameters 2104 ---------- 2105 None 2106 2107 Returns 2108 ------- 2109 int 2110 The number of epochs since the last switch, or total epochs run, 2111 depending on settings. 2112 2113 """ 2114 if self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_TOTAL_EPOCH: 2115 return self.member_vars["num_epochs_run"] 2116 elif ( 2117 self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_UPDATE_EPOCH 2118 ): 2119 return self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2120 elif ( 2121 self.member_vars["param_vals_setting"] 2122 == GPA.pc.PARAM_VALS_BY_NEURON_EPOCH_START 2123 ): 2124 if self.member_vars["mode"] == "p": 2125 return ( 2126 self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2127 ) 2128 else: 2129 return self.member_vars["num_epochs_run"] 2130 else: 2131 print( 2132 f'{self.member_vars["param_vals_setting"]} is not a valid param vals option' 2133 ) 2134 pdb.set_trace() 2135 2136 def add_pai_neuron_module(self, new_module, initial_add=True): 2137 """Add neuron modules to internal vectors. 2138 2139 Parameters 2140 ---------- 2141 new_module : object 2142 The new module to add. 2143 initial_add : bool, optional 2144 Whether this is the initial addition rather than loading from file 2145 2146 Returns 2147 ------- 2148 None 2149 2150 """ 2151 2152 # If it's a duplicate, ignore the second addition 2153 if new_module in self.neuron_module_vector: 2154 return 2155 self.neuron_module_vector.append(new_module) 2156 if self.member_vars["doing_pai"]: 2157 PA.set_wrapped_params(new_module) 2158 if initial_add: 2159 self.member_vars["best_scores"].append([]) 2160 self.member_vars["current_scores"].append([]) 2161 2162 def add_tracked_neuron_module(self, new_module, initial_add=True): 2163 """Add tracked modules to internal vectors 2164 2165 Parameters 2166 ---------- 2167 new_module : object 2168 The new module to add. 2169 initial_add : bool, optional 2170 Whether this is the initial addition rather than loading from file 2171 2172 Returns 2173 ------- 2174 None 2175 2176 """ 2177 # If it's a duplicate, ignore the second addition 2178 if new_module in self.tracked_neuron_module_vector: 2179 return 2180 self.tracked_neuron_module_vector.append(new_module) 2181 if self.member_vars["doing_pai"]: 2182 PA.set_tracked_params(new_module) 2183 2184 def reset_module_vector(self, net, load_from_restart): 2185 """Clear internal vectors and reset from network. 2186 2187 Parameters 2188 ---------- 2189 net : object 2190 The neural network model. 2191 load_from_restart : bool 2192 Whether loading from a restart file. 2193 2194 Returns 2195 ------- 2196 None 2197 2198 """ 2199 self.neuron_module_vector = [] 2200 self.tracked_neuron_module_vector = [] 2201 this_list = UPA.get_pai_modules(net, 0) 2202 for module in this_list: 2203 self.add_pai_neuron_module(module, initial_add=load_from_restart) 2204 this_list = UPA.get_tracked_modules(net, 0) 2205 for module in this_list: 2206 self.add_tracked_neuron_module(module, initial_add=load_from_restart) 2207 2208 def reset_vals_for_score_reset(self): 2209 """Reset cycle scores for new cycle. 2210 2211 Parameters 2212 ---------- 2213 None 2214 2215 Returns 2216 ------- 2217 None 2218 This function does not return a value. 2219 """ 2220 2221 if GPA.pc.get_find_best_lr(): 2222 self.member_vars["committed_to_initial_rate"] = False 2223 print("Resetting committed to initial rate to False") 2224 # If retaining all dendrties always say that the current dendrites set global best for saving and loading 2225 if GPA.pc.get_retain_all_dendrites(): 2226 self.member_vars["current_n_set_global_best"] = True 2227 self.member_vars["global_best_validation_score"] = 0 2228 else: 2229 self.member_vars["current_n_set_global_best"] = False 2230 2231 # Don't reset global best, but do reset current best 2232 self.member_vars["current_best_validation_score"] = 0 2233 self.member_vars["initial_lr_test_epoch_count"] = -1 2234 2235 def set_dendrite_training(self): 2236 """Signal all layers to start dendrite training. 2237 2238 Parameters 2239 ---------- 2240 None 2241 2242 Returns 2243 ------- 2244 None 2245 This function does not return a value. 2246 """ 2247 if GPA.pc.get_verbose(): 2248 print("Calling set_dendrite_training") 2249 2250 for layer in self.neuron_module_vector[:]: 2251 worked = layer.set_mode("p") 2252 """ 2253 worked is False when a layer was added to the neuron module vector 2254 but then it's never actually been used. This can happen when 2255 you have set a layer to have requires_grad = False or when 2256 you have a module as a member variable but it's not actually 2257 part of the network. Should be moved to be a tracked layer 2258 rather than a neuron layer. 2259 """ 2260 if not worked: 2261 self.neuron_module_vector.remove(layer) 2262 2263 for layer in self.tracked_neuron_module_vector[:]: 2264 worked = layer.set_mode("p") 2265 2266 self.create_new_dendrite_module() 2267 self.member_vars["mode"] = "p" 2268 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2269 2270 if GPA.pc.get_learn_dendrites_live(): 2271 self.reset_vals_for_score_reset() 2272 2273 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2274 "current_step_count" 2275 ] 2276 2277 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 2278 GPA.pai_tracker.member_vars["num_cycles"] += 1 2279 2280 2281 def set_neuron_training(self): 2282 """Signal all layers to start neuron training. 2283 2284 Parameters 2285 ---------- 2286 None 2287 2288 Returns 2289 ------- 2290 None 2291 This function does not return a value. 2292 """ 2293 for module in self.neuron_module_vector: 2294 module.set_mode("n") 2295 for module in self.tracked_neuron_module_vector[:]: 2296 module.set_mode("n") 2297 2298 self.member_vars["mode"] = "n" 2299 self.member_vars["num_dendrites_added"] += 1 2300 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2301 self.reset_vals_for_score_reset() 2302 2303 self.member_vars["current_cycle_lr_max_scores"] = [] 2304 if GPA.pc.get_learn_dendrites_live(): 2305 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2306 "current_step_count" 2307 ] 2308 GPA.pai_tracker.member_vars["num_cycles"] += 1 2309 2310 if GPA.pc.get_reset_best_score_on_switch(): 2311 GPA.pai_tracker.member_vars["current_best_validation_score"] = 0 2312 GPA.pai_tracker.member_vars["running_accuracy"] = 0 2313 2314 def start_epoch(self, internal_call=False): 2315 """Perform steps for when a new training epoch is about to begin. 2316 2317 Parameters 2318 ---------- 2319 internal_call : bool, optional 2320 Whether this is an internal call or manual call 2321 2322 Returns 2323 ------- 2324 None 2325 2326 Notes 2327 ----- 2328 If you ever need to call this manually, set internal_call to False. 2329 2330 """ 2331 if self.member_vars["manual_train_switch"] and internal_call: 2332 return 2333 2334 if not internal_call and not self.member_vars["manual_train_switch"]: 2335 self.member_vars["manual_train_switch"] = True 2336 self.saved_time = 0 2337 self.member_vars["num_epochs_run"] = -1 2338 self.member_vars["total_epochs_run"] = -1 2339 2340 end = time.time() 2341 if self.member_vars["manual_train_switch"]: 2342 if self.saved_time != 0: 2343 if self.member_vars["mode"] == "p": 2344 self.member_vars["p_val_times"].append(end - self.saved_time) 2345 else: 2346 self.member_vars["n_val_times"].append(end - self.saved_time) 2347 2348 if self.member_vars["mode"] == "p": 2349 for layer in self.neuron_module_vector: 2350 for m in range(0, GPA.pc.get_global_candidates()): 2351 with torch.no_grad(): 2352 if GPA.pc.get_verbose(): 2353 print(f"Resetting score for {layer.name}") 2354 # Snapshot best_score before reset so we can compute per-epoch improvement 2355 layer.dendrite_module.dendrite_values[ 2356 m 2357 ].epoch_start_best_score.copy_( 2358 layer.dendrite_module.dendrite_values[ 2359 m 2360 ].best_score.detach() 2361 ) 2362 layer.dendrite_module.dendrite_values[ 2363 m 2364 ].best_score_improved_this_epoch = ( 2365 layer.dendrite_module.dendrite_values[ 2366 m 2367 ].best_score_improved_this_epoch 2368 * 0 2369 ) 2370 layer.dendrite_module.dendrite_values[ 2371 m 2372 ].nodes_best_improved_this_epoch = ( 2373 layer.dendrite_module.dendrite_values[ 2374 m 2375 ].nodes_best_improved_this_epoch 2376 * 0 2377 ) 2378 layer.dendrite_module.dendrite_values[ 2379 m 2380 ].nodes_improved_any = ( 2381 layer.dendrite_module.dendrite_values[ 2382 m 2383 ].nodes_improved_any 2384 * 0 2385 ) 2386 if GPA.pc.get_perforated_backpropagation(): 2387 self.member_vars["best_mean_score_improved_this_epoch"] = 0 2388 self.member_vars["num_epochs_run"] += 1 2389 self.member_vars["total_epochs_run"] = ( 2390 self.member_vars["num_epochs_run"] + self.member_vars["overwritten_epochs"] 2391 ) 2392 self.saved_time = end 2393 2394 def stop_epoch(self, internal_call=False): 2395 """Perform steps when a training epoch has completed. 2396 2397 Parameters 2398 ---------- 2399 internal_call : bool, optional 2400 Whether this is an internal call or manual call 2401 2402 Returns 2403 ------- 2404 None 2405 2406 Notes 2407 ----- 2408 If you ever need to call this manually, set internal_call to False. 2409 2410 """ 2411 end = time.time() 2412 if self.member_vars["manual_train_switch"] and internal_call: 2413 return 2414 2415 if self.member_vars["manual_train_switch"]: 2416 if self.member_vars["mode"] == "p": 2417 self.member_vars["p_train_times"].append(end - self.saved_time) 2418 else: 2419 self.member_vars["n_train_times"].append(end - self.saved_time) 2420 else: 2421 if self.member_vars["mode"] == "p": 2422 self.member_vars["p_epoch_times"].append(end - self.saved_time) 2423 else: 2424 self.member_vars["n_epoch_times"].append(end - self.saved_time) 2425 2426 self.saved_time = end 2427 2428 def initialize( 2429 self, 2430 model, 2431 doing_pai=True, 2432 save_name="PAI", 2433 making_graphs=True, 2434 maximizing_score=True, 2435 num_classes=10000, 2436 values_per_train_epoch=-1, 2437 values_per_val_epoch=-1, 2438 zooming_graph=True, 2439 ): 2440 """Setup the tracker with initial settings. 2441 2442 2443 Parameters 2444 ---------- 2445 model : object 2446 The neural network model. 2447 doing_pai : bool, optional 2448 Whether to add dendrites, by default True. 2449 save_name : str, optional 2450 The name under which to save the model. 2451 making_graphs : bool, optional 2452 Whether to make graphs, by default True. 2453 maximizing_score : bool, optional 2454 Whether to maximize the score, by default True. 2455 num_classes : int, optional 2456 The number of classes in the dataset, unused 2457 values_per_train_epoch : int, optional 2458 The number of values to look back for graphing 2459 during training, by default -1 (all values). 2460 values_per_val_epoch : int, optional 2461 The number of values to look back for graphing 2462 during validation, by default -1 (all values). 2463 zooming_graph : bool, optional 2464 Whether to zoom on graphs, by default True. 2465 2466 2467 Returns 2468 ------- 2469 nn.Module 2470 Converted model instance configured for the tracker settings. 2471 """ 2472 model = UPA.convert_network(model) 2473 self.member_vars["doing_pai"] = doing_pai 2474 self.member_vars["maximizing_score"] = maximizing_score 2475 self.save_name = save_name 2476 self.zooming_graph = zooming_graph 2477 self.making_graphs = making_graphs 2478 2479 if not self.loaded: 2480 self.member_vars["running_accuracy"] = (1.0 / num_classes) * 100 2481 2482 self.values_per_train_epoch = values_per_train_epoch 2483 self.values_per_val_epoch = values_per_val_epoch 2484 2485 if GPA.pc.get_testing_dendrite_capacity(): 2486 if not GPA.pc.get_silent(): 2487 print("Running a test of Dendrite Capacity.") 2488 GPA.pc.set_switch_mode(GPA.pc.DOING_SWITCH_EVERY_TIME) 2489 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 2490 GPA.pc.set_retain_all_dendrites(True) 2491 GPA.pc.set_max_dendrite_tries(1000) 2492 GPA.pc.set_max_dendrites(1000) 2493 if GPA.pc.get_perforated_backpropagation(): 2494 GPA.pc.set_initial_correlation_batches(1) 2495 else: 2496 if not GPA.pc.get_silent(): 2497 print("Running Dendrite Experiment") 2498 return model 2499 2500 def generate_accuracy_plots(self, ax, save_folder, extra_string): 2501 """ 2502 Generate plots and csvs for accuracy 2503 2504 Parameters 2505 ---------- 2506 ax : object 2507 The matplotlib axis to plot on. 2508 save_folder : str 2509 The folder to save the plots and csvs in. 2510 extra_string : str 2511 An extra string to append to the filenames. 2512 2513 Returns 2514 ------- 2515 None 2516 2517 """ 2518 2519 # If scores are being saved for epochs that get overwritten, plot them 2520 for list_id in range(len(self.member_vars["overwritten_extras"])): 2521 for extra_id in self.member_vars["overwritten_extras"][list_id]: 2522 ax.plot( 2523 np.arange( 2524 len(self.member_vars["overwritten_extras"][list_id][extra_id]) 2525 ), 2526 self.member_vars["overwritten_extras"][list_id][extra_id], 2527 color=_PAI_PALETTE["train"], 2528 alpha=0.35, 2529 ) 2530 ax.plot( 2531 np.arange(len(self.member_vars["overwritten_vals"][list_id])), 2532 self.member_vars["overwritten_vals"][list_id], 2533 color=_PAI_PALETTE["validation"], 2534 alpha=0.35, 2535 ) 2536 2537 # Determine which accuracy vector to use 2538 if GPA.pc.get_drawing_pai(): 2539 accuracies = self.member_vars["accuracies"] 2540 else: 2541 accuracies = self.member_vars["n_accuracies"] 2542 2543 # Get pointer to additional scores being saved 2544 extra_scores = self.member_vars["extra_scores"] 2545 2546 # Plot the main accuracy scores 2547 ax.plot( 2548 np.arange(len(accuracies)), 2549 accuracies, 2550 label="Validation Scores", 2551 color=_PAI_PALETTE["validation"], 2552 ) 2553 ax.plot( 2554 np.arange(len(self.member_vars["running_accuracies"])), 2555 self.member_vars["running_accuracies"], 2556 label="Validation Running Scores", 2557 color=_PAI_PALETTE["validation"], 2558 alpha=0.4, 2559 ) 2560 2561 # Plot additional scores 2562 for extra_index, extra_score in enumerate(extra_scores): 2563 ax.plot( 2564 np.arange(len(extra_scores[extra_score])), 2565 extra_scores[extra_score], 2566 label=extra_score, 2567 color=_pai_extra_score_color(extra_score, extra_index), 2568 ) 2569 2570 plt.title(save_folder + "/" + self.save_name + "Scores") 2571 plt.xlabel("Epochs") 2572 plt.ylabel("Score") 2573 2574 # Add point at epoch last improved and best validation score 2575 if GPA.pc.get_drawing_pai(): 2576 ax.plot( 2577 self.member_vars["epoch_last_improved"], 2578 self.member_vars["global_best_validation_score"], 2579 "o", 2580 color=_PAI_PALETTE["marker_best"], 2581 label="Global best (y)", 2582 ) 2583 ax.plot( 2584 self.member_vars["epoch_last_improved"], 2585 accuracies[self.member_vars["epoch_last_improved"]], 2586 "o", 2587 color=_PAI_PALETTE["marker_improved"], 2588 label="Epoch Last Improved", 2589 ) 2590 else: 2591 if self.member_vars["mode"] == "n": 2592 missed_time = ( 2593 self.member_vars["num_epochs_run"] 2594 - self.member_vars["epoch_last_improved"] 2595 ) 2596 ax.plot( 2597 (len(self.member_vars["n_accuracies"]) - 1) - missed_time, 2598 self.member_vars["n_accuracies"][-(missed_time + 1)], 2599 "o", 2600 color=_PAI_PALETTE["marker_improved"], 2601 label="Epoch Last Improved", 2602 ) 2603 2604 # Generate csv file for the values graphed 2605 pd1 = pd.DataFrame( 2606 {"Epochs": np.arange(len(accuracies)), "Validation Scores": accuracies} 2607 ) 2608 pd2 = pd.DataFrame( 2609 { 2610 "Epochs": np.arange(len(self.member_vars["running_accuracies"])), 2611 "Validation Running Scores": self.member_vars["running_accuracies"], 2612 } 2613 ) 2614 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2615 for extra_score in extra_scores: 2616 pd2 = pd.DataFrame( 2617 { 2618 "Epochs": np.arange(len(extra_scores[extra_score])), 2619 extra_score: extra_scores[extra_score], 2620 } 2621 ) 2622 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2623 extra_scores_without_graphing = self.member_vars[ 2624 "extra_scores_without_graphing" 2625 ] 2626 for extra_score in extra_scores_without_graphing: 2627 pd2 = pd.DataFrame( 2628 { 2629 "Epochs": np.arange( 2630 len(extra_scores_without_graphing[extra_score]) 2631 ), 2632 extra_score: extra_scores_without_graphing[extra_score], 2633 } 2634 ) 2635 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2636 pd1.to_csv( 2637 save_folder + "/" + self.save_name + extra_string + "Scores.csv", 2638 index=False, 2639 ) 2640 del pd1, pd2 2641 2642 # Set y min and max to zoom in on important part of axis 2643 if ( 2644 len(self.member_vars["switch_epochs"]) > 0 2645 and self.member_vars["switch_epochs"][0] > 0 2646 and self.zooming_graph 2647 ): 2648 if GPA.pai_tracker.member_vars["maximizing_score"]: 2649 min_val = np.array( 2650 accuracies[0 : self.member_vars["switch_epochs"][0]] 2651 ).mean() 2652 for extra_score in extra_scores: 2653 min_pot = np.array( 2654 extra_scores[extra_score][ 2655 0 : self.member_vars["switch_epochs"][0] 2656 ] 2657 ).mean() 2658 if min_pot < min_val: 2659 min_val = min_pot 2660 ax.set_ylim(ymin=min_val) 2661 else: 2662 max_val = np.array( 2663 accuracies[0 : self.member_vars["switch_epochs"][0]] 2664 ).mean() 2665 for extra_score in extra_scores: 2666 max_pot = np.array( 2667 extra_scores[extra_score][ 2668 0 : self.member_vars["switch_epochs"][0] 2669 ] 2670 ).mean() 2671 if max_pot > max_val: 2672 max_val = max_pot 2673 ax.set_ylim(ymax=max_val) 2674 2675 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2676 _pai_grid(ax) 2677 2678 # Draw vertical lines for epochs where a dendrite switch occurred. 2679 # Colour by the phase being entered: dendrite/PAI phase -> teal, 2680 # normal phase -> orange, alternating starting with dendrite. 2681 if GPA.pc.get_drawing_pai() and self.member_vars["doing_pai"]: 2682 for i, switcher in enumerate(self.member_vars["switch_epochs"]): 2683 # Mode always starts at "n" and switch_epochs alternates 2684 # n->p->n->p..., so the first switch enters the dendrite 2685 # phase and every other one alternates from there. 2686 entering_p = (i % 2) == 0 2687 plt.axvline( 2688 x=switcher, 2689 ymin=0, 2690 ymax=1, 2691 color=_PAI_PALETTE["dendrite"] 2692 if entering_p 2693 else _PAI_PALETTE["normal"], 2694 ) 2695 else: 2696 for switcher in self.member_vars["n_switch_epochs"]: 2697 plt.axvline( 2698 x=switcher, ymin=0, ymax=1, color=_PAI_PALETTE["normal"] 2699 ) 2700 2701 def generate_time_plots(self, ax, save_folder, extra_string): 2702 """ 2703 Generate plots and csvs for timing 2704 2705 Parameters 2706 ---------- 2707 ax : object 2708 The matplotlib axis to plot on. 2709 save_folder : str 2710 The folder to save the plots and csvs in. 2711 extra_string : str 2712 An extra string to append to the filenames. 2713 2714 Returns 2715 ------- 2716 None 2717 2718 """ 2719 if self.member_vars["manual_train_switch"]: 2720 ax.plot( 2721 np.arange(len(self.member_vars["n_train_times"])), 2722 self.member_vars["n_train_times"], 2723 label="Normal Epoch Train Times", 2724 color=_PAI_PALETTE["normal"], 2725 ) 2726 ax.plot( 2727 np.arange(len(self.member_vars["p_train_times"])), 2728 self.member_vars["p_train_times"], 2729 label="PAI Epoch Train Times", 2730 color=_PAI_PALETTE["dendrite"], 2731 ) 2732 ax.plot( 2733 np.arange(len(self.member_vars["n_val_times"])), 2734 self.member_vars["n_val_times"], 2735 label="Normal Epoch Val Times", 2736 color=_PAI_PALETTE["normal"], 2737 alpha=0.45, 2738 ) 2739 ax.plot( 2740 np.arange(len(self.member_vars["p_val_times"])), 2741 self.member_vars["p_val_times"], 2742 label="PAI Epoch Val Times", 2743 color=_PAI_PALETTE["dendrite"], 2744 alpha=0.45, 2745 ) 2746 2747 plt.title( 2748 save_folder + "/" + self.save_name + "times (by train() and eval())" 2749 ) 2750 plt.xlabel("Iteration") 2751 plt.ylabel("Epoch Time in Seconds ") 2752 ax.set_ylim(ymin=0) 2753 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2754 _pai_grid(ax) 2755 2756 pd1 = pd.DataFrame( 2757 { 2758 "Epochs": np.arange(len(self.member_vars["n_train_times"])), 2759 "Normal Epoch Train Times": self.member_vars["n_train_times"], 2760 } 2761 ) 2762 pd2 = pd.DataFrame( 2763 { 2764 "Epochs": np.arange(len(self.member_vars["p_train_times"])), 2765 "PAI Epoch Train Times": self.member_vars["p_train_times"], 2766 } 2767 ) 2768 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2769 2770 pd2 = pd.DataFrame( 2771 { 2772 "Epochs": np.arange(len(self.member_vars["n_val_times"])), 2773 "Normal Epoch Val Times": self.member_vars["n_val_times"], 2774 } 2775 ) 2776 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2777 2778 pd2 = pd.DataFrame( 2779 { 2780 "Epochs": np.arange(len(self.member_vars["p_val_times"])), 2781 "PAI Epoch Val Times": self.member_vars["p_val_times"], 2782 } 2783 ) 2784 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2785 2786 pd1.to_csv( 2787 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2788 index=False, 2789 ) 2790 del pd1, pd2 2791 else: 2792 ax.plot( 2793 np.arange(len(self.member_vars["n_epoch_times"])), 2794 self.member_vars["n_epoch_times"], 2795 label="Normal Epoch Times", 2796 color=_PAI_PALETTE["normal"], 2797 ) 2798 ax.plot( 2799 np.arange(len(self.member_vars["p_epoch_times"])), 2800 self.member_vars["p_epoch_times"], 2801 label="PAI Epoch Times", 2802 color=_PAI_PALETTE["dendrite"], 2803 ) 2804 2805 plt.title( 2806 save_folder + "/" + self.save_name + "times (by train() and eval())" 2807 ) 2808 plt.xlabel("Iteration") 2809 plt.ylabel("Epoch Time in Seconds ") 2810 ax.set_ylim(ymin=0) 2811 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2812 _pai_grid(ax) 2813 2814 pd1 = pd.DataFrame( 2815 { 2816 "Epochs": np.arange(len(self.member_vars["n_epoch_times"])), 2817 "Normal Epoch Times": self.member_vars["n_epoch_times"], 2818 } 2819 ) 2820 pd2 = pd.DataFrame( 2821 { 2822 "Epochs": np.arange(len(self.member_vars["p_epoch_times"])), 2823 "PAI Epoch Times": self.member_vars["p_epoch_times"], 2824 } 2825 ) 2826 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2827 2828 pd1.to_csv( 2829 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2830 index=False, 2831 ) 2832 del pd1, pd2 2833 2834 if self.values_per_train_epoch != -1 and self.values_per_val_epoch != -1: 2835 ax2 = ax.twinx() # Second axes sharing same x-axis 2836 ax2.set_ylabel("Single Datapoint Time in Seconds") 2837 2838 ax2.plot( 2839 np.arange(len(self.member_vars["n_train_times"])), 2840 np.array(self.member_vars["n_train_times"]) 2841 / self.values_per_train_epoch, 2842 linestyle="dashed", 2843 label="Normal Train Item Times", 2844 color=_PAI_PALETTE["normal"], 2845 ) 2846 ax2.plot( 2847 np.arange(len(self.member_vars["p_train_times"])), 2848 np.array(self.member_vars["p_train_times"]) 2849 / self.values_per_train_epoch, 2850 linestyle="dashed", 2851 label="PAI Train Item Times", 2852 color=_PAI_PALETTE["dendrite"], 2853 ) 2854 ax2.plot( 2855 np.arange(len(self.member_vars["n_val_times"])), 2856 np.array(self.member_vars["n_val_times"]) / self.values_per_val_epoch, 2857 linestyle="dashed", 2858 label="Normal Val Item Times", 2859 color=_PAI_PALETTE["normal"], 2860 alpha=0.45, 2861 ) 2862 ax2.plot( 2863 np.arange(len(self.member_vars["p_val_times"])), 2864 np.array(self.member_vars["p_val_times"]) / self.values_per_val_epoch, 2865 linestyle="dashed", 2866 label="PAI Val Item Times", 2867 color=_PAI_PALETTE["dendrite"], 2868 alpha=0.45, 2869 ) 2870 ax2.tick_params(axis="y") 2871 ax2.set_ylim(ymin=0) 2872 ax2.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2873 2874 def generate_learning_rate_plots(self, ax, save_folder, extra_string): 2875 """ 2876 Generate plots and csvs for learning rate 2877 2878 Parameters 2879 ---------- 2880 ax : object 2881 The matplotlib axis to plot on. 2882 save_folder : str 2883 The folder to save the plots and csvs in. 2884 extra_string : str 2885 An extra string to append to the filenames. 2886 2887 Returns 2888 ------- 2889 None 2890 2891 """ 2892 ax.plot( 2893 np.arange(len(self.member_vars["training_learning_rates"])), 2894 self.member_vars["training_learning_rates"], 2895 label="learning_rate", 2896 color=_PAI_PALETTE["learning_rate"], 2897 ) 2898 plt.title(save_folder + "/" + self.save_name + "learning_rate") 2899 plt.xlabel("Epochs") 2900 plt.ylabel("learning_rate") 2901 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2902 _pai_grid(ax) 2903 2904 pd1 = pd.DataFrame( 2905 { 2906 "Epochs": np.arange(len(self.member_vars["training_learning_rates"])), 2907 "learning_rate": self.member_vars["training_learning_rates"], 2908 } 2909 ) 2910 pd1.to_csv( 2911 save_folder + "/" + self.save_name + extra_string + "learning_rate.csv", 2912 index=False, 2913 ) 2914 del pd1 2915 2916 def get_current_pb_scores(self): 2917 """ 2918 Get the latest best PBScore of each dendrite layer, the same numbers 2919 written to the Best PBScores csv. 2920 2921 Returns 2922 ------- 2923 dict 2924 Layer name to score. Empty outside of dendrite scoring phases, 2925 when no candidate dendrites are being scored. 2926 2927 """ 2928 if not self.member_vars["doing_pai"]: 2929 return {} 2930 if not GPA.pc.get_perforated_backpropagation(): 2931 return {} 2932 # Scores only advance while candidate dendrites are being trained 2933 if ( 2934 self.member_vars["mode"] != "p" 2935 and not GPA.pc.get_learn_dendrites_live() 2936 ): 2937 return {} 2938 2939 scores = {} 2940 for layer_id in range(len(self.neuron_module_vector)): 2941 if layer_id >= len(self.member_vars["best_scores"]): 2942 continue 2943 layer_scores = self.member_vars["best_scores"][layer_id] 2944 if len(layer_scores) == 0: 2945 continue 2946 score = layer_scores[-1] 2947 if hasattr(score, "item"): 2948 score = score.item() 2949 score = float(score) 2950 if math.isnan(score) or math.isinf(score): 2951 continue 2952 scores[self.neuron_module_vector[layer_id].name] = score 2953 return scores 2954 2955 def get_current_pb_scores_current(self): 2956 """ 2957 Like :meth:`get_current_pb_scores` but returns the *current* (not best) 2958 PBScore of each dendrite layer - ``current_scores[layer][-1]`` - the 2959 dashed series in ``generate_dendrite_learning_plots``. 2960 2961 Returns 2962 ------- 2963 dict 2964 Layer name to score. Empty outside of dendrite scoring phases. 2965 """ 2966 if not self.member_vars["doing_pai"]: 2967 return {} 2968 if not GPA.pc.get_perforated_backpropagation(): 2969 return {} 2970 if ( 2971 self.member_vars["mode"] != "p" 2972 and not GPA.pc.get_learn_dendrites_live() 2973 ): 2974 return {} 2975 2976 scores = {} 2977 for layer_id in range(len(self.neuron_module_vector)): 2978 if layer_id >= len(self.member_vars["current_scores"]): 2979 continue 2980 layer_scores = self.member_vars["current_scores"][layer_id] 2981 if len(layer_scores) == 0: 2982 continue 2983 score = layer_scores[-1] 2984 if hasattr(score, "item"): 2985 score = score.item() 2986 score = float(score) 2987 if math.isnan(score) or math.isinf(score): 2988 continue 2989 scores[self.neuron_module_vector[layer_id].name] = score 2990 return scores 2991 2992 def generate_dendrite_learning_plots(self, ax, save_folder, extra_string): 2993 """ 2994 Generate dendrite score plots for the tracker. 2995 Also saves csv files associated with the plots. 2996 2997 Parameters 2998 ---------- 2999 ax : matplotlib.axes.Axes 3000 Axis used for plotting dendrite-learning curves. 3001 save_folder : str 3002 Directory where plot images and CSV summaries are written. 3003 extra_string : str 3004 Filename suffix used to distinguish this output set. 3005 3006 Returns 3007 ------- 3008 None 3009 Saves plots and score CSV files to disk. 3010 """ 3011 if self.member_vars["doing_pai"]: 3012 pd1 = None 3013 pd2 = None 3014 num_colors = len(self.neuron_module_vector) 3015 3016 layer_colors = [ 3017 _PAI_DENDRITE_CYCLE[i % len(_PAI_DENDRITE_CYCLE)] 3018 for i in range(num_colors) 3019 ] 3020 3021 for layer_id in range(len(self.neuron_module_vector)): 3022 color = layer_colors[layer_id] 3023 ax.plot( 3024 np.arange(len(self.member_vars["best_scores"][layer_id])), 3025 self.member_vars["best_scores"][layer_id], 3026 label=self.neuron_module_vector[layer_id].name, 3027 color=color, 3028 ) 3029 3030 pd2 = pd.DataFrame( 3031 { 3032 "Epochs": np.arange( 3033 len(self.member_vars["best_scores"][layer_id]) 3034 ), 3035 f"Best ever for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 3036 "best_scores" 3037 ][ 3038 layer_id 3039 ], 3040 } 3041 ) 3042 3043 if pd1 is None: 3044 pd1 = pd2 3045 else: 3046 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 3047 3048 if len(self.member_vars["current_scores"][layer_id]) != 0: 3049 ax.plot( 3050 np.arange(len(self.member_vars["current_scores"][layer_id])), 3051 self.member_vars["current_scores"][layer_id], 3052 label=f"Current:{self.neuron_module_vector[layer_id].name}", 3053 color=color, 3054 linestyle="--", 3055 ) 3056 3057 pd2 = pd.DataFrame( 3058 { 3059 "Epochs": np.arange( 3060 len(self.member_vars["current_scores"][layer_id]) 3061 ), 3062 f"Best current for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 3063 "current_scores" 3064 ][ 3065 layer_id 3066 ], 3067 } 3068 ) 3069 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 3070 3071 plt.title(save_folder + "/" + self.save_name + " Best PBScores") 3072 plt.xlabel("Epochs") 3073 plt.ylabel("Best PBScore") 3074 ax.legend( 3075 bbox_to_anchor=(1.05, 1), 3076 loc="upper left", 3077 ncol=max(1, math.ceil(len(self.neuron_module_vector) / 30)), 3078 ) 3079 _pai_grid(ax) 3080 for switcher in self.member_vars["p_switch_epochs"]: 3081 plt.axvline( 3082 x=switcher, ymin=0, ymax=1, color=_PAI_PALETTE["dendrite"] 3083 ) 3084 3085 if self.member_vars["mode"] == "p": 3086 missed_time = ( 3087 self.member_vars["num_epochs_run"] 3088 - self.member_vars["epoch_last_improved"] 3089 ) 3090 plt.axvline( 3091 x=(len(self.member_vars["best_scores"][0]) - (missed_time + 1)), 3092 ymin=0, 3093 ymax=1, 3094 color=_PAI_PALETTE["normal"], 3095 ) 3096 3097 # pd1 here will be none if no PB layers are created 3098 if pd1 is not None: 3099 pd1.to_csv( 3100 save_folder 3101 + "/" 3102 + self.save_name 3103 + extra_string 3104 + "Best PBScores.csv", 3105 index=False, 3106 ) 3107 del pd1, pd2 3108 3109 def generate_extra_csv_files(self, save_folder, extra_string): 3110 """ 3111 Generate additional csvs 3112 3113 Parameters 3114 ---------- 3115 save_folder : str 3116 The folder to save the plots and csvs in. 3117 extra_string : str 3118 An extra string to append to the filenames. 3119 3120 Returns 3121 ------- 3122 None 3123 3124 """ 3125 pd1 = pd.DataFrame( 3126 { 3127 "Switch Number": np.arange(len(self.member_vars["switch_epochs"])), 3128 "Switch Epoch": self.member_vars["switch_epochs"], 3129 } 3130 ) 3131 pd1.to_csv( 3132 save_folder + "/" + self.save_name + extra_string + "switch_epochs.csv", 3133 index=False, 3134 ) 3135 del pd1 3136 3137 pd1 = pd.DataFrame( 3138 { 3139 "Switch Number": np.arange(len(self.member_vars["param_counts"])), 3140 "Param Count": self.member_vars["param_counts"], 3141 } 3142 ) 3143 pd1.to_csv( 3144 save_folder + "/" + self.save_name + extra_string + "param_counts.csv", 3145 index=False, 3146 ) 3147 del pd1 3148 3149 """ 3150 Create best_arch_scores.csv file 3151 When working with dendrites there is a tradeoff between additional param count and score improvement. 3152 This file will help track that tradeoff by recording the best scores for all extra_scores 3153 and extra_scores_without_graphing for each architecture version. 3154 The scores recorded here are from the epoch when the best validation score was found 3155 within each switch_epoch boundary. 3156 """ 3157 switch_counts = len(self.member_vars["switch_epochs"]) 3158 best_valid = [] 3159 associated_params = [] 3160 3161 # Initialize dictionaries to store best scores for each extra score type 3162 best_extra_scores = {} 3163 for score_name in self.member_vars["extra_scores"]: 3164 best_extra_scores[score_name] = [] 3165 for score_name in self.member_vars["extra_scores_without_graphing"]: 3166 best_extra_scores[score_name] = [] 3167 3168 for switch in range(0, switch_counts, 2): 3169 start_index = 0 3170 if switch != 0: 3171 start_index = self.member_vars["switch_epochs"][switch - 1] + 1 3172 end_index = self.member_vars["switch_epochs"][switch] + 1 3173 3174 if GPA.pai_tracker.member_vars["maximizing_score"]: 3175 best_valid_index = start_index + np.argmax( 3176 self.member_vars["accuracies"][start_index:end_index] 3177 ) 3178 else: 3179 best_valid_index = start_index + np.argmin( 3180 self.member_vars["accuracies"][start_index:end_index] 3181 ) 3182 3183 best_valid_score = self.member_vars["accuracies"][best_valid_index] 3184 best_valid.append(best_valid_score) 3185 3186 # Get corresponding scores from all extra_scores 3187 for score_name in self.member_vars["extra_scores"]: 3188 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 3189 best_extra_scores[score_name].append( 3190 self.member_vars["extra_scores"][score_name][best_valid_index] 3191 ) 3192 else: 3193 best_extra_scores[score_name].append(None) 3194 3195 # Get corresponding scores from all extra_scores_without_graphing 3196 for score_name in self.member_vars["extra_scores_without_graphing"]: 3197 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 3198 best_extra_scores[score_name].append( 3199 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 3200 ) 3201 else: 3202 best_extra_scores[score_name].append(None) 3203 3204 if self.member_vars["doing_pai"]: 3205 associated_params.append(self.member_vars["param_counts"][switch]) 3206 else: 3207 associated_params.append(self.member_vars["param_counts"][-1]) 3208 3209 # If in neuron training mode but not the very first epoch 3210 if self.member_vars["mode"] == "n" and ( 3211 (len(self.member_vars["switch_epochs"]) == 0) 3212 or ( 3213 self.member_vars["switch_epochs"][-1] + 1 3214 != len(self.member_vars["accuracies"]) 3215 ) 3216 ): 3217 start_index = 0 3218 if len(self.member_vars["switch_epochs"]) != 0: 3219 start_index = self.member_vars["switch_epochs"][-1] + 1 3220 3221 if GPA.pai_tracker.member_vars["maximizing_score"]: 3222 best_valid_index = start_index + np.argmax( 3223 self.member_vars["accuracies"][start_index:] 3224 ) 3225 else: 3226 best_valid_index = start_index + np.argmin( 3227 self.member_vars["accuracies"][start_index:] 3228 ) 3229 3230 best_valid_score = self.member_vars["accuracies"][best_valid_index] 3231 best_valid.append(best_valid_score) 3232 3233 # Get corresponding scores from all extra_scores 3234 for score_name in self.member_vars["extra_scores"]: 3235 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 3236 best_extra_scores[score_name].append( 3237 self.member_vars["extra_scores"][score_name][best_valid_index] 3238 ) 3239 else: 3240 best_extra_scores[score_name].append(None) 3241 3242 # Get corresponding scores from all extra_scores_without_graphing 3243 for score_name in self.member_vars["extra_scores_without_graphing"]: 3244 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 3245 best_extra_scores[score_name].append( 3246 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 3247 ) 3248 else: 3249 best_extra_scores[score_name].append(None) 3250 3251 associated_params.append(self.member_vars["param_counts"][-1]) 3252 3253 # Build dataframe with all columns 3254 csv_data = { 3255 "Param Counts": associated_params, 3256 "Max Valid Scores": best_valid, 3257 } 3258 3259 # Add columns for each extra score 3260 for score_name in best_extra_scores: 3261 csv_data[score_name] = best_extra_scores[score_name] 3262 3263 pd1 = pd.DataFrame(csv_data) 3264 pd1.to_csv( 3265 save_folder + "/" + self.save_name + extra_string + "_best_arch_scores.csv", 3266 index=False, 3267 ) 3268 del pd1 3269 3270 def save_graphs(self, extra_string=""): 3271 """ 3272 Save graphs and csvs for all the values the tracker records 3273 3274 Parameters 3275 ---------- 3276 extra_string : str 3277 An extra string to append to the filenames. 3278 3279 Returns 3280 ------- 3281 None 3282 3283 """ 3284 # If running DDP only save with rank 0 3285 if "RANK" in os.environ: 3286 if int(os.environ["RANK"]) != 0: 3287 return 3288 if not self.making_graphs: 3289 return 3290 3291 save_folder = "./" + self.save_name + "/" 3292 3293 plt.ioff() 3294 fig = plt.figure(figsize=(28, 14)) 3295 3296 # Plot with accuracy scores 3297 ax = plt.subplot(221) 3298 self.generate_accuracy_plots(ax, save_folder, extra_string) 3299 3300 # Plot dendrite learning scores 3301 ax = plt.subplot(222) 3302 self.generate_dendrite_learning_plots(ax, save_folder, extra_string) 3303 3304 if GPA.pc.get_drawing_extra_graphs(): 3305 # Plot learning rates for each training epoch 3306 ax = plt.subplot(223) 3307 self.generate_learning_rate_plots(ax, save_folder, extra_string) 3308 3309 # Plot the times for each training epoch 3310 ax = plt.subplot(224) 3311 self.generate_time_plots(ax, save_folder, extra_string) 3312 3313 # Generate extra CSV files 3314 self.generate_extra_csv_files(save_folder, extra_string) 3315 3316 fig.tight_layout() 3317 plt.savefig(save_folder + "/" + self.save_name + extra_string + ".png") 3318 plt.close("all") 3319 3320 def add_loss(self, loss): 3321 """Add loss to tracking vectors. 3322 3323 Parameters 3324 ---------- 3325 loss : float or int 3326 The loss value to add. 3327 3328 Returns 3329 ------- 3330 None 3331 3332 """ 3333 if not isinstance(loss, (float, int)): 3334 loss = loss.item() 3335 self.member_vars["training_loss"].append(loss) 3336 3337 def add_learning_rate(self, learning_rate): 3338 """Add learning rate to tracking vectors. 3339 3340 Parameters 3341 ---------- 3342 learning_rate : float or int 3343 The learning rate value to add. 3344 3345 Returns 3346 ------- 3347 None 3348 3349 """ 3350 if not isinstance(learning_rate, (float, int)): 3351 learning_rate = learning_rate.item() 3352 self.member_vars["training_learning_rates"].append(learning_rate) 3353 3354 def add_extra_score(self, score, extra_score_name): 3355 """Add extra score to tracking vectors. 3356 3357 Parameters 3358 ---------- 3359 score : float or int 3360 The score value to add. 3361 3362 extra_score_name : str 3363 The name of the extra score. 3364 3365 Returns 3366 ------- 3367 None 3368 3369 """ 3370 if not isinstance(score, (float, int)): 3371 try: 3372 score = score.item() 3373 except: 3374 print( 3375 "Scores added for Perforated Backpropagation should be " 3376 "float, int, or tensor, yours is a:" 3377 ) 3378 print(type(score)) 3379 pdb.set_trace() 3380 3381 if GPA.pc.get_verbose(): 3382 print(f"Adding extra score {extra_score_name} of {float(score)}") 3383 3384 if extra_score_name not in self.member_vars["extra_scores"]: 3385 self.member_vars["extra_scores"][extra_score_name] = [] 3386 self.member_vars["extra_scores"][extra_score_name].append(score) 3387 3388 if self.member_vars["mode"] == "n": 3389 if extra_score_name not in self.member_vars["n_extra_scores"]: 3390 self.member_vars["n_extra_scores"][extra_score_name] = [] 3391 self.member_vars["n_extra_scores"][extra_score_name].append(score) 3392 3393 def add_extra_score_without_graphing(self, score, extra_score_name): 3394 """Add extra score without graphing to tracking vectors. 3395 3396 Parameters 3397 ---------- 3398 score : float or int 3399 The score value to add. 3400 3401 extra_score_name : str 3402 The name of the extra score. 3403 3404 Returns 3405 ------- 3406 None 3407 3408 """ 3409 if not isinstance(score, (float, int)): 3410 try: 3411 score = score.item() 3412 except: 3413 print( 3414 "Scores added for Perforated Backpropagation should be " 3415 "float, int, or tensor, yours is a:" 3416 ) 3417 print(type(score)) 3418 print("in add_extra_score_without_graphing") 3419 pdb.set_trace() 3420 3421 if GPA.pc.get_verbose(): 3422 print(f"Adding extra score {extra_score_name} of {float(score)}") 3423 3424 if extra_score_name not in self.member_vars["extra_scores_without_graphing"]: 3425 self.member_vars["extra_scores_without_graphing"][extra_score_name] = [] 3426 self.member_vars["extra_scores_without_graphing"][extra_score_name].append( 3427 score 3428 ) 3429 3430 def add_test_score(self, score, extra_score_name): 3431 """Add test score to tracking vectors. 3432 3433 Parameters 3434 ---------- 3435 score : float or int 3436 The score value to add. 3437 3438 extra_score_name : str 3439 The name of the extra score. 3440 3441 Returns 3442 ------- 3443 None 3444 3445 Notes 3446 ----- 3447 This function is a wrapper around `add_extra_score` that separates 3448 test score for adding to best_arch_scores.csv. 3449 3450 """ 3451 self.add_extra_score(score, extra_score_name) 3452 3453 if not isinstance(score, (float, int)): 3454 try: 3455 score = score.item() 3456 except: 3457 print( 3458 "Scores added for Perforated Backpropagation should be " 3459 "float, int, or tensor, yours is a:" 3460 ) 3461 print(type(score)) 3462 print("in add_test_score") 3463 pdb.set_trace() 3464 3465 if GPA.pc.get_verbose(): 3466 print(f"Adding test score {extra_score_name} of {float(score)}") 3467 self.member_vars["test_scores"].append(score) 3468 3469 def add_validation_score(self, accuracy, net, force_switch=False): 3470 """Function to add the validation score. 3471 3472 This is complex because it determines neuron and dendrite switching. 3473 3474 Parameters 3475 ---------- 3476 accuracy : float or int 3477 The accuracy or loss value to add. 3478 net : object 3479 The neural network model. 3480 force_switch : bool, optional 3481 Whether to force a switch, by default False. 3482 3483 Returns 3484 ------- 3485 net : object 3486 The potentially modified neural network model. 3487 training_complete : bool 3488 Whether training is complete. 3489 restructured : bool 3490 Whether the model has been restructured. 3491 3492 Notes 3493 ----- 3494 WARNING: Do not call self anywhere in this function. When systems 3495 get loaded the actual tracker you are working with can change. 3496 """ 3497 3498 _pai_log("info", f"Adding validation score {accuracy:.8f}") 3499 3500 update_learning_rate() 3501 update_param_count(net) 3502 3503 accuracy = check_input_problems(net, accuracy) 3504 3505 if len(GPA.pai_tracker.member_vars["switch_epochs"]) == 0: 3506 epochs_since_cycle_switch = GPA.pai_tracker.member_vars["num_epochs_run"] 3507 else: 3508 epochs_since_cycle_switch = ( 3509 GPA.pai_tracker.member_vars["num_epochs_run"] 3510 - GPA.pai_tracker.member_vars["switch_epochs"][-1] 3511 ) 3512 3513 update_running_accuracy(accuracy, epochs_since_cycle_switch) 3514 if GPA.pc.get_perforated_backpropagation(): 3515 TPB.update_pb_scores(self) 3516 3517 # Captured before any switch below flips the mode and reloads scores. 3518 # epoch_phase is sampled here too so the emitted epoch event's phase and 3519 # its pb_scores always describe the same side of a switch boundary. 3520 epoch_phase = GPA.pai_tracker.member_vars["mode"] 3521 epoch_pb_scores = self.get_current_pb_scores() 3522 epoch_pb_scores_current = self.get_current_pb_scores_current() 3523 3524 GPA.pai_tracker.stop_epoch(internal_call=True) 3525 3526 # If it is neuron training mode 3527 if ( 3528 GPA.pai_tracker.member_vars["mode"] == "n" 3529 or GPA.pc.get_learn_dendrites_live() 3530 ): 3531 check_new_best(net, accuracy, epochs_since_cycle_switch) 3532 elif GPA.pc.get_perforated_backpropagation(): 3533 TPB.check_best_pai_score_improvement() 3534 3535 # Save the latest model 3536 if GPA.pc.get_test_saves(): 3537 UPA.save_system(net, GPA.pc.get_save_name(), "latest") 3538 if GPA.pc.get_pai_saves(): 3539 UPA.pai_save_system(net, GPA.pc.get_save_name(), "latest") 3540 3541 restructuring_status_value = NO_MODEL_UPDATE 3542 # If it is time to switch based on scores and counter or a manual switch 3543 if GPA.pai_tracker.switch_time() or force_switch: 3544 # If testing dendrite capacity switch after enough dendrites added 3545 if ( 3546 (GPA.pai_tracker.member_vars["mode"] == "n") 3547 and (GPA.pai_tracker.member_vars["num_dendrites_added"] > 2) 3548 and GPA.pc.get_testing_dendrite_capacity() 3549 ): 3550 GPA.pai_tracker.save_graphs() 3551 _pai_log( 3552 "info", 3553 "Successfully added 3 dendrites with GPA.pc.set_testing_dendrite_capacity(True) (default). " 3554 "You may now set that to False and run a real experiment.", 3555 ) 3556 return net, False, True 3557 3558 # If doing neuron training but this dendrite count didn't improve 3559 if ( 3560 (GPA.pai_tracker.member_vars["mode"] == "n") 3561 or GPA.pc.get_learn_dendrites_live() 3562 ) and (GPA.pai_tracker.member_vars["current_n_set_global_best"] is False): 3563 new_restructuring_status_value, net = process_no_improvement(net) 3564 # if this was the final try return that training is complete 3565 if new_restructuring_status_value == TRAINING_COMPLETE: 3566 if _dashboard_emitter is not None: 3567 _dashboard_emitter.emit_run_end( 3568 GPA.pc, 3569 epoch_last_improved=GPA.pai_tracker.member_vars[ 3570 "epoch_last_improved" 3571 ], 3572 global_best_score=GPA.pai_tracker.member_vars[ 3573 "global_best_validation_score" 3574 ], 3575 ) 3576 return net, True, True 3577 else: 3578 restructuring_status_value = update_restructuring_status( 3579 restructuring_status_value, new_restructuring_status_value 3580 ) 3581 # Else if did improve, do a normal switch process 3582 else: 3583 if GPA.pc.get_verbose(): 3584 print( 3585 f"Calling switch_mode with " 3586 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]}, ' 3587 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 3588 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]}, ' 3589 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_value"]},' 3590 f'{GPA.pc.get_max_dendrites()},' 3591 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]},' 3592 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"]},' 3593 ) 3594 import pdb; pdb.set_trace 3595 # If the max number of dendrites has been hit or not doing pai and adding dendtites 3596 # then return rather than adding more 3597 if ( 3598 (GPA.pai_tracker.member_vars["mode"] == "n") 3599 and ( 3600 GPA.pc.get_max_dendrites() 3601 == GPA.pai_tracker.member_vars["num_dendrites_added"] 3602 ) 3603 ) or (GPA.pai_tracker.member_vars["doing_pai"] is False): 3604 if GPA.pc.get_verbose(): 3605 print( 3606 "Max dendrites reached or not doing PAI, finishing training" 3607 ) 3608 net = process_final_network(net) 3609 # Increment integrated if we have dendrites (means they're integrated) 3610 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3611 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3612 _pai_log("info", f"Final dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3613 if _dashboard_emitter is not None: 3614 _dashboard_emitter.emit_dendrite_added( 3615 GPA.pc, 3616 epoch_index=len( 3617 GPA.pai_tracker.member_vars["accuracies"] 3618 ) - 1, 3619 num_dendrites_integrated=GPA.pai_tracker.member_vars[ 3620 "num_dendrites_integrated" 3621 ], 3622 param_count=UPA.count_params(net), 3623 ) 3624 if _dashboard_emitter is not None: 3625 _dashboard_emitter.emit_run_end( 3626 GPA.pc, 3627 epoch_last_improved=GPA.pai_tracker.member_vars[ 3628 "epoch_last_improved" 3629 ], 3630 global_best_score=GPA.pai_tracker.member_vars[ 3631 "global_best_validation_score" 3632 ], 3633 ) 3634 return net, True, True 3635 3636 # Otherwise if its neuron training mode reset the counter of failed dendrites 3637 # Check if we should increment integrated count BEFORE change_learning_modes loads old state 3638 should_increment_integrated = False 3639 if GPA.pai_tracker.member_vars["mode"] == "n": 3640 GPA.pai_tracker.member_vars["num_dendrite_tries"] = 0 3641 if GPA.pc.get_verbose(): 3642 print( 3643 "Adding new dendrites without resetting which means " 3644 "the last ones improved. Resetting num_dendrite_tries" 3645 ) 3646 # Remember to increment after change_learning_modes (which loads old tracker state) 3647 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3648 should_increment_integrated = True 3649 3650 GPA.pai_tracker.save_graphs( 3651 f'_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}' 3652 ) 3653 3654 if GPA.pc.get_test_saves(): 3655 UPA.save_system( 3656 net, 3657 GPA.pc.get_save_name(), 3658 f'beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3659 ) 3660 # Copy current best model from this set of dendrites 3661 # If running DDP only copy with rank 0 3662 if "RANK" not in os.environ or int(os.environ["RANK"]) == 0: 3663 shutil.copyfile( 3664 f"{GPA.pc.get_save_name()}/best_model.pt", 3665 f'{GPA.pc.get_save_name()}/best_model_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}.pt', 3666 ) 3667 3668 if GPA.pc.get_dashboard_debug(): 3669 _pai_log( 3670 "debug", 3671 "[emitter/finding4] before change_learning_modes: " 3672 f'len(accuracies)={len(GPA.pai_tracker.member_vars["accuracies"])} ' 3673 f'num_epochs_run={GPA.pai_tracker.member_vars["num_epochs_run"]}', 3674 ) 3675 net = UPA.change_learning_modes( 3676 net, 3677 GPA.pc.get_save_name(), 3678 "best_model", 3679 GPA.pai_tracker.member_vars["doing_pai"], 3680 ) 3681 restructuring_status_value = NETWORK_RESTRUCTURED 3682 if GPA.pc.get_dashboard_debug(): 3683 _pai_log( 3684 "debug", 3685 "[emitter/finding4] after change_learning_modes: " 3686 f'len(accuracies)={len(GPA.pai_tracker.member_vars["accuracies"])} ' 3687 f'num_epochs_run={GPA.pai_tracker.member_vars["num_epochs_run"]}', 3688 ) 3689 3690 # Now increment after change_learning_modes has loaded the best model 3691 # This ensures the increment persists and doesn't get overwritten 3692 if should_increment_integrated: 3693 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3694 _pai_log("info", f"Dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3695 if _dashboard_emitter is not None: 3696 _dashboard_emitter.emit_dendrite_added( 3697 GPA.pc, 3698 epoch_index=len( 3699 GPA.pai_tracker.member_vars["accuracies"] 3700 ) - 1, 3701 num_dendrites_integrated=GPA.pai_tracker.member_vars[ 3702 "num_dendrites_integrated" 3703 ], 3704 param_count=UPA.count_params(net), 3705 ) 3706 3707 # If restructured is true, clear scheduler/optimizer before saving 3708 if restructuring_status_value != NETWORK_RESTRUCTURED: 3709 print( 3710 "Restructured should always be triggered here, let us know if you encounter this situation" 3711 ) 3712 pdb.set_trace() 3713 3714 # Since there is a restructuring optimizer and scheduler must be reinitialized after return 3715 GPA.pai_tracker.clear_optimizer_and_scheduler() 3716 3717 # Save the model from after the switch 3718 UPA.save_system( 3719 net, 3720 GPA.pc.get_save_name(), 3721 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3722 ) 3723 3724 # If not time to switch and you have a scheduler, perform the update step 3725 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 3726 new_restructuring_status_value, net = process_scheduler_update( 3727 net, accuracy, epochs_since_cycle_switch 3728 ) 3729 restructuring_status_value = update_restructuring_status( 3730 restructuring_status_value, new_restructuring_status_value 3731 ) 3732 3733 GPA.pai_tracker.start_epoch(internal_call=True) 3734 if _dashboard_emitter is not None: 3735 _mv = GPA.pai_tracker.member_vars 3736 _lr = _mv["training_learning_rates"][-1] if _mv["training_learning_rates"] else None 3737 # epoch_index is the committed coordinate: len(accuracies) - 1, 3738 # captured after update_running_accuracy appends and after any reload 3739 # truncation in switch handling. Independent of num_epochs_run. 3740 _epoch_index = len(_mv["accuracies"]) - 1 3741 if GPA.pc.get_dashboard_debug(): 3742 _pai_log( 3743 "debug", 3744 "[emitter/finding4] emit_epoch: " 3745 f"epoch_index={_epoch_index} " 3746 f'true_epoch={_mv["num_epochs_run"]} ' 3747 f'restructured={restructuring_status_value == NETWORK_RESTRUCTURED} ' 3748 f'tail_accuracies={_mv["accuracies"][-4:]}', 3749 ) 3750 # Phase captured at the top of the method (pre-switch), matching the 3751 # point pb_scores were sampled - see Finding 2, ticket 06. 3752 _phase = epoch_phase 3753 # scores: validation (reported), running (checkpointed member var), 3754 # then every add_extra_score name verbatim. 3755 _scores = { 3756 "validation": accuracy, 3757 "running": _mv["running_accuracies"][-1] if _mv["running_accuracies"] else None, 3758 } 3759 for _name, _vals in _mv["extra_scores"].items(): 3760 if _vals: 3761 _scores[_name] = _vals[-1] 3762 _n_times = _mv["n_epoch_times"] or [(_mv["n_train_times"][-1] + _mv["n_val_times"][-1]) if (_mv["n_train_times"] and _mv["n_val_times"]) else None] 3763 _p_times = _mv["p_epoch_times"] or [(_mv["p_train_times"][-1] + _mv["p_val_times"][-1]) if (_mv["p_train_times"] and _mv["p_val_times"]) else None] 3764 _dashboard_emitter.emit_epoch( 3765 GPA.pc, 3766 epoch_index=_epoch_index, 3767 phase=_phase, 3768 scores=_scores, 3769 learning_rate=_lr, 3770 normal_time=_n_times[-1] if _phase == "n" else None, 3771 pai_time=_p_times[-1] if _phase == "p" else None, 3772 pb_scores=epoch_pb_scores, 3773 pb_scores_current=epoch_pb_scores_current, 3774 ) 3775 GPA.pai_tracker.save_graphs() 3776 3777 if restructuring_status_value == NETWORK_RESTRUCTURED: 3778 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 3779 GPA.pai_tracker.member_vars["num_epochs_run"] 3780 ) 3781 if GPA.pc.get_verbose(): 3782 print( 3783 f"Setting epoch last improved to " 3784 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]}' 3785 ) 3786 3787 now = datetime.now() 3788 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 3789 3790 if GPA.pc.get_verbose(): 3791 print("Not saving restructure right now") 3792 3793 """ 3794 This block of code helped with a save issue with safetensors and huggingface, but it breaks DDP. 3795 Temporarily removing it to avoid DDP issues, but if you encounter save issues try adding it back in. 3796 for param in net.parameters(): 3797 param.data = param.data.contiguous() 3798 """ 3799 if GPA.pc.get_verbose(): 3800 print( 3801 f"Completed adding score. Restructured is {restructuring_status_value}, " 3802 f"\ncurrent switch list is:" 3803 ) 3804 print(GPA.pai_tracker.member_vars["switch_epochs"]) 3805 3806 if _dashboard_emitter is not None and restructuring_status_value == NETWORK_RESTRUCTURED: 3807 _param_count = UPA.count_params(net) 3808 _dashboard_emitter.emit_switch( 3809 GPA.pc, 3810 # switch_epochs was already appended in change_learning_modes, 3811 # so subtract 1 for a 0-based ordinal (matches matplotlib's 3812 # np.arange(len(switch_epochs)) labelling) - Finding 1, ticket 06. 3813 switch_ordinal=len(GPA.pai_tracker.member_vars["switch_epochs"]) - 1, 3814 epoch_index=len(GPA.pai_tracker.member_vars["accuracies"]) - 1, 3815 param_count=_param_count, 3816 switch_type=GPA.pai_tracker.member_vars["mode"], 3817 ) 3818 3819 # Always False for training complete if nothing triggered that training is over 3820 return net, restructuring_status_value, False 3821 3822 def clear_all_processors(self): 3823 """Clear all processors from modules. 3824 3825 Parameters 3826 ---------- 3827 None 3828 3829 Returns 3830 ------- 3831 None 3832 This function does not return a value. 3833 """ 3834 for module in self.neuron_module_vector: 3835 module.clear_processors() 3836 3837 def create_new_dendrite_module(self): 3838 """Add dendrite module to all neuron modules. 3839 3840 Parameters 3841 ---------- 3842 None 3843 3844 Returns 3845 ------- 3846 None 3847 This function does not return a value. 3848 """ 3849 for module in self.neuron_module_vector: 3850 module.create_new_dendrite_module() 3851 3852 def set_create_dendrite_global(self, fn): 3853 """Call set_create_dendrite(fn) on every tracked PAINeuronModule.""" 3854 for module in self.neuron_module_vector: 3855 module.set_create_dendrite(fn) 3856 3857 def set_dendrite_loss_fn_global(self, fn): 3858 """Set the global dendrite loss function used by all dendrite modules.""" 3859 from perforatedbp import modules_pbp as MPB 3860 MPB.dendrite_loss_fn = fn 3861 3862 def apply_pb_grads(self): 3863 """Apply perforated backpropagation gradients to all modules. 3864 3865 Parameters 3866 ---------- 3867 None 3868 3869 Returns 3870 ------- 3871 None 3872 This function does not return a value. 3873 """ 3874 if self.member_vars["mode"] == "p": 3875 for module in self.neuron_module_vector: 3876 module.apply_pb_grads() 3877 3878 def apply_pb_zero(self): 3879 """Apply perforated backpropagation zero gradients to all modules. 3880 3881 Parameters 3882 ---------- 3883 None 3884 3885 Returns 3886 ------- 3887 None 3888 This function does not return a value. 3889 """ 3890 if self.member_vars["mode"] == "p": 3891 for module in self.neuron_module_vector: 3892 module.apply_pb_zero()
Manager class that tracks all neuron layers and dendrite layers, controls when new dendrites are added, and communicates signals to modules.
PAINeuronModuleTracker( doing_pai, save_name, making_graphs=True, param_vals_setting=-1, values_per_train_epoch=-1, values_per_val_epoch=-1)
990 def __init__( 991 self, 992 doing_pai, 993 save_name, 994 making_graphs=True, 995 param_vals_setting=-1, 996 values_per_train_epoch=-1, 997 values_per_val_epoch=-1, 998 ): 999 """Initialize the tracker 1000 1001 Parameters 1002 ---------- 1003 doing_pai : bool 1004 Whether or not dendrites should be used. 1005 save_name : str 1006 The base name for saving models and graphs. 1007 making_graphs : bool, optional 1008 Whether or not to generate graphs, by default True. 1009 param_vals_setting : int, optional 1010 Parameter values setting, by default -1. 1011 values_per_train_epoch : int, optional 1012 The number of values to look back for graphing 1013 during training, by default -1 (all values). 1014 values_per_val_epoch : int, optional 1015 The number of values to look back for graphing 1016 during validation, by default -1 (all values). 1017 Returns 1018 ------- 1019 None 1020 """ 1021 1022 # Dict of member vars and their types for saving 1023 self.member_vars = {} 1024 self.member_var_types = {} 1025 1026 # Whether or not PAI will be running 1027 self.member_vars["doing_pai"] = doing_pai 1028 self.member_var_types["doing_pai"] = "bool" 1029 1030 # How many Dendrites have been added 1031 self.member_vars["num_dendrites_added"] = 0 1032 self.member_var_types["num_dendrites_added"] = "int" 1033 1034 # How many Dendrites have been successfully integrated, does not count currently training dendrites 1035 self.member_vars["num_dendrites_integrated"] = 0 1036 self.member_var_types["num_dendrites_integrated"] = "int" 1037 1038 # How many cycles have been run, *2 or *2+1 of the above 1039 self.member_vars["num_cycles"] = 0 1040 self.member_var_types["num_cycles"] = "int" 1041 1042 # Pointers to all neuron wrapped modules 1043 self.neuron_module_vector = [] 1044 1045 # Pointers to all non neuron modules for tracking 1046 self.tracked_neuron_module_vector = [] 1047 1048 # Neuron training or dendrite training mode 1049 self.member_vars["mode"] = "n" 1050 self.member_var_types["mode"] = "string" 1051 1052 # Number of epochs run excluding overwritten epochs 1053 self.member_vars["num_epochs_run"] = -1 1054 self.member_var_types["num_epochs_run"] = "int" 1055 1056 # Number including overwritten epochs 1057 self.member_vars["total_epochs_run"] = -1 1058 self.member_var_types["total_epochs_run"] = "int" 1059 1060 # Last epoch that validation/correlation score was improved 1061 self.member_vars["epoch_last_improved"] = 0 1062 self.member_var_types["epoch_last_improved"] = "int" 1063 1064 # Running validation accuracy 1065 self.member_vars["running_accuracy"] = 0 1066 self.member_var_types["running_accuracy"] = "float" 1067 1068 # True if maxing validation, False if minimizing Loss 1069 self.member_vars["maximizing_score"] = True 1070 self.member_var_types["maximizing_score"] = "bool" 1071 1072 # Mode for switching back and forth between learning modes 1073 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 1074 self.member_var_types["switch_mode"] = "int" 1075 1076 # Epoch of the last switch 1077 self.member_vars["last_switch"] = 0 1078 self.member_var_types["last_switch"] = "int" 1079 1080 # Highest validation score from current cycle 1081 self.member_vars["current_best_validation_score"] = 0 1082 self.member_var_types["current_best_validation_score"] = "float" 1083 1084 # Last epoch where the learning rate was updated 1085 self.member_vars["initial_lr_test_epoch_count"] = -1 1086 self.member_var_types["initial_lr_test_epoch_count"] = "int" 1087 1088 # Highest validation score of full run 1089 self.member_vars["global_best_validation_score"] = 0 1090 self.member_var_types["global_best_validation_score"] = "float" 1091 1092 # List of switch epochs 1093 self.member_vars["switch_epochs"] = [] 1094 self.member_var_types["switch_epochs"] = "int array" 1095 1096 # Parameter counts at each network structure 1097 self.member_vars["param_counts"] = [] 1098 self.member_var_types["param_counts"] = "int array" 1099 1100 # List of epochs where switch was made to neuron training 1101 self.member_vars["n_switch_epochs"] = [] 1102 self.member_var_types["n_switch_epochs"] = "int array" 1103 1104 # List of epochs where switch was made to dendrite training 1105 self.member_vars["p_switch_epochs"] = [] 1106 self.member_var_types["p_switch_epochs"] = "int array" 1107 1108 # List of validation accuracies 1109 self.member_vars["accuracies"] = [] 1110 self.member_var_types["accuracies"] = "float array" 1111 1112 # List of epochs where score improved for scheduler updates 1113 self.member_vars["last_improved_accuracies"] = [] 1114 self.member_var_types["last_improved_accuracies"] = "int array" 1115 1116 # List of test accuracy scores registered 1117 self.member_vars["test_accuracies"] = [] 1118 self.member_var_types["test_accuracies"] = "float array" 1119 1120 # List of accuracies registered during neuron training 1121 self.member_vars["n_accuracies"] = [] 1122 self.member_var_types["n_accuracies"] = "float array" 1123 1124 # List of accuracies registered during dendrite training 1125 self.member_vars["p_accuracies"] = [] 1126 self.member_var_types["p_accuracies"] = "float array" 1127 1128 # Running average accuracies from recent epochs 1129 self.member_vars["running_accuracies"] = [] 1130 self.member_var_types["running_accuracies"] = "float array" 1131 1132 # List of additional scores recorded 1133 self.member_vars["extra_scores"] = {} 1134 self.member_var_types["extra_scores"] = "float array dictionary" 1135 1136 # Extra scores not set to be graphed 1137 self.member_vars["extra_scores_without_graphing"] = {} 1138 self.member_var_types["extra_scores_without_graphing"] = ( 1139 "float array dictionary" 1140 ) 1141 1142 # List of test scores 1143 self.member_vars["test_scores"] = [] 1144 self.member_var_types["test_scores"] = "float array" 1145 1146 # Extra scores calculated during neuron training 1147 self.member_vars["n_extra_scores"] = {} 1148 self.member_var_types["n_extra_scores"] = "float array dictionary" 1149 1150 # List of training losses calculated 1151 self.member_vars["training_loss"] = [] 1152 self.member_var_types["training_loss"] = "float array" 1153 1154 # List of learning rates at each epoch 1155 self.member_vars["training_learning_rates"] = [] 1156 self.member_var_types["training_learning_rates"] = "float array" 1157 1158 # Best dendrite scores 1159 self.member_vars["best_scores"] = [] 1160 self.member_var_types["best_scores"] = "float array array" 1161 1162 # Current dendrite scores 1163 self.member_vars["current_scores"] = [] 1164 self.member_var_types["current_scores"] = "float array array" 1165 1166 # Times for neuron training epochs 1167 self.member_vars["n_epoch_times"] = [] 1168 self.member_var_types["n_epoch_times"] = "float array" 1169 1170 # Timing values 1171 self.member_vars["p_epoch_times"] = [] 1172 self.member_var_types["p_epoch_times"] = "float array" 1173 self.member_vars["n_train_times"] = [] 1174 self.member_var_types["n_train_times"] = "float array" 1175 self.member_vars["p_train_times"] = [] 1176 self.member_var_types["p_train_times"] = "float array" 1177 self.member_vars["n_val_times"] = [] 1178 self.member_var_types["n_val_times"] = "float array" 1179 self.member_vars["p_val_times"] = [] 1180 self.member_var_types["p_val_times"] = "float array" 1181 1182 # Setting for tracking timing 1183 self.member_vars["manual_train_switch"] = False 1184 self.member_var_types["manual_train_switch"] = "bool" 1185 1186 # Tracking scores overwritten when reloading best model 1187 self.member_vars["overwritten_extras"] = [] 1188 self.member_var_types["overwritten_extras"] = "float array dictionary array" 1189 self.member_vars["overwritten_vals"] = [] 1190 self.member_var_types["overwritten_vals"] = "float array array" 1191 self.member_vars["overwritten_epochs"] = 0 1192 self.member_var_types["overwritten_epochs"] = "int" 1193 1194 # Setting for determining scores 1195 self.member_vars["param_vals_setting"] = GPA.pc.get_param_vals_setting() 1196 self.member_var_types["param_vals_setting"] = "int" 1197 1198 # Optimizer and scheduler types and instances 1199 self.member_vars["optimizer"] = None 1200 self.member_var_types["optimizer"] = "type" 1201 self.member_vars["scheduler"] = None 1202 self.member_var_types["scheduler"] = "type" 1203 self.member_vars["optimizer_instance"] = None 1204 self.member_var_types["optimizer_instance"] = "empty array" 1205 self.member_vars["scheduler_instance"] = None 1206 self.member_var_types["scheduler_instance"] = "empty array" 1207 1208 # Flag for if the tracker was loaded 1209 self.loaded = False 1210 1211 # flag for 1212 self.member_vars["step_status"] = STEP_CLEARED 1213 self.member_var_types["step_status"] = "int" 1214 1215 1216 # Settings for tracking learning rates 1217 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 1218 self.member_var_types["current_n_learning_rate_initial_skip_steps"] = "int" 1219 self.member_vars["last_max_learning_rate_steps"] = 0 1220 self.member_var_types["last_max_learning_rate_steps"] = "int" 1221 self.member_vars["last_max_learning_rate_value"] = -1 1222 self.member_var_types["last_max_learning_rate_value"] = "float" 1223 self.member_vars["current_cycle_lr_max_scores"] = [] 1224 self.member_var_types["current_cycle_lr_max_scores"] = "float array" 1225 self.member_vars["current_step_count"] = 0 1226 self.member_var_types["current_step_count"] = "int" 1227 self.member_vars["committed_to_initial_rate"] = True 1228 self.member_var_types["committed_to_initial_rate"] = "bool" 1229 self.member_vars["best_mean_score_improved_this_epoch"] = 0 1230 self.member_var_types["best_mean_score_improved_this_epoch"] = "int" 1231 1232 # Flag for if current dendrite achieved highest global score 1233 self.member_vars["current_n_set_global_best"] = True 1234 self.member_var_types["current_n_set_global_best"] = "bool" 1235 1236 # Number of tries adding this dendrite count 1237 self.member_vars["num_dendrite_tries"] = 0 1238 self.member_var_types["num_dendrite_tries"] = "int" 1239 1240 # Count of batches per epoch 1241 self.values_per_train_epoch = values_per_train_epoch 1242 self.values_per_val_epoch = values_per_val_epoch 1243 1244 self.save_name = save_name 1245 self.making_graphs = making_graphs 1246 1247 self.start_time = time.time() 1248 self.saved_time = 0 1249 self.start_epoch(internal_call=True) 1250 1251 if GPA.pc.get_verbose(): 1252 print(f'Initializing with switch_mode {self.member_vars["switch_mode"]}')
Initialize the tracker
Parameters
- doing_pai (bool): Whether or not dendrites should be used.
- save_name (str): The base name for saving models and graphs.
- making_graphs (bool, optional): Whether or not to generate graphs, by default True.
- param_vals_setting (int, optional): Parameter values setting, by default -1.
- values_per_train_epoch (int, optional): The number of values to look back for graphing during training, by default -1 (all values).
- values_per_val_epoch (int, optional): The number of values to look back for graphing during validation, by default -1 (all values).
Returns
- None
def
to_string(self):
1254 def to_string(self): 1255 """Convert tracker values to string for saving with safetensors. 1256 1257 Parameters 1258 ---------- 1259 None 1260 1261 Returns 1262 ------- 1263 str 1264 Serialized tracker state suitable for storage in a safetensors field. 1265 """ 1266 1267 full_string = "" 1268 for var in self.member_vars: 1269 full_string += var + "," 1270 if self.member_vars[var] is None: 1271 full_string += "None" 1272 full_string += "\n" 1273 elif self.member_var_types[var] == "bool": 1274 full_string += str(self.member_vars[var]) 1275 full_string += "\n" 1276 elif self.member_var_types[var] in ("int", "float", "string"): 1277 full_string += str(self.member_vars[var]) 1278 full_string += "\n" 1279 elif self.member_var_types[var] == "type": 1280 name = ( 1281 self.member_vars[var].__module__ 1282 + "." 1283 + self.member_vars[var].__name__ 1284 ) 1285 full_string += str(self.member_vars[var]) 1286 full_string += "\n" 1287 elif self.member_var_types[var] == "empty array": 1288 full_string += "[]" 1289 full_string += "\n" 1290 elif self.member_var_types[var] in ("int array", "float array"): 1291 full_string += "\n" 1292 string = "" 1293 for val in self.member_vars[var]: 1294 string += str(val) + "," 1295 # Remove the last comma 1296 string = string[:-1] 1297 full_string += string 1298 full_string += "\n" 1299 elif self.member_var_types[var] == "float array dictionary array": 1300 full_string += "\n" 1301 for array in self.member_vars[var]: 1302 for key in array: 1303 string = key + "," 1304 for val in array[key]: 1305 string += str(val) + "," 1306 # Remove the last comma 1307 string = string[:-1] 1308 full_string += string 1309 full_string += "\n" 1310 full_string += "endkey" 1311 full_string += "\n" 1312 full_string += "endarray" 1313 full_string += "\n" 1314 elif self.member_var_types[var] == "float array dictionary": 1315 full_string += "\n" 1316 for key in self.member_vars[var]: 1317 string = key + "," 1318 for val in self.member_vars[var][key]: 1319 string += str(val) + "," 1320 # Remove the last comma 1321 string = string[:-1] 1322 full_string += string 1323 full_string += "\n" 1324 full_string += "end" 1325 full_string += "\n" 1326 elif self.member_var_types[var] == "float array array": 1327 full_string += "\n" 1328 for array in self.member_vars[var]: 1329 string = "" 1330 for val in array: 1331 string += str(val) + "," 1332 # Remove the last comma 1333 string = string[:-1] 1334 full_string += string 1335 full_string += "\n" 1336 full_string += "end" 1337 full_string += "\n" 1338 else: 1339 print("Did not find a member variable") 1340 pdb.set_trace() 1341 return full_string
Convert tracker values to string for saving with safetensors.
Parameters
- None
Returns
- str: Serialized tracker state suitable for storage in a safetensors field.
def
from_string(self, string):
1343 def from_string(self, string): 1344 """Load tracker values from string. 1345 1346 Parameters 1347 ---------- 1348 string : str 1349 The string to load from. 1350 1351 Returns 1352 ------- 1353 None 1354 This function does not return a value. 1355 """ 1356 f = io.StringIO(string) 1357 while True: 1358 line = f.readline() 1359 if not line: 1360 break 1361 vals = line.split(",") 1362 var = vals[0] 1363 1364 if self.member_var_types[var] == "bool": 1365 val = vals[1][:-1] 1366 if val == "True": 1367 self.member_vars[var] = True 1368 elif val == "False": 1369 self.member_vars[var] = False 1370 elif val == "1": 1371 self.member_vars[var] = 1 1372 elif val == "0": 1373 self.member_vars[var] = 0 1374 else: 1375 print("Something went wrong with loading") 1376 pdb.set_trace() 1377 elif self.member_var_types[var] == "int": 1378 val = vals[1] 1379 self.member_vars[var] = int(val) 1380 elif self.member_var_types[var] == "float": 1381 val = vals[1] 1382 self.member_vars[var] = float(val) 1383 elif self.member_var_types[var] == "string": 1384 val = vals[1][:-1] 1385 self.member_vars[var] = val 1386 elif self.member_var_types[var] == "type": 1387 # Ignore loading types, tracker should have them set up 1388 continue 1389 elif self.member_var_types[var] == "empty array": 1390 val = vals[1] 1391 self.member_vars[var] = [] 1392 elif self.member_var_types[var] == "int array": 1393 vals = f.readline()[:-1].split(",") 1394 self.member_vars[var] = [] 1395 if vals[0] == "": 1396 continue 1397 for val in vals: 1398 self.member_vars[var].append(int(val)) 1399 elif self.member_var_types[var] == "float array": 1400 vals = f.readline()[:-1].split(",") 1401 self.member_vars[var] = [] 1402 if vals[0] == "": 1403 continue 1404 for val in vals: 1405 self.member_vars[var].append(float(val)) 1406 elif self.member_var_types[var] == "float array dictionary array": 1407 self.member_vars[var] = [] 1408 line2 = f.readline()[:-1] 1409 while line2 != "endarray": 1410 temp = {} 1411 while line2 != "endkey": 1412 vals = line2.split(",") 1413 name = vals[0] 1414 temp[name] = [] 1415 vals = vals[1:] 1416 for val in vals: 1417 temp[name].append(float(val)) 1418 line2 = f.readline()[:-1] 1419 self.member_vars[var].append(temp) 1420 line2 = f.readline()[:-1] 1421 elif self.member_var_types[var] == "float array dictionary": 1422 self.member_vars[var] = {} 1423 line2 = f.readline()[:-1] 1424 while line2 != "end": 1425 vals = line2.split(",") 1426 name = vals[0] 1427 self.member_vars[var][name] = [] 1428 vals = vals[1:] 1429 for val in vals: 1430 self.member_vars[var][name].append(float(val)) 1431 line2 = f.readline()[:-1] 1432 elif self.member_var_types[var] == "float array array": 1433 self.member_vars[var] = [] 1434 line2 = f.readline()[:-1] 1435 while line2 != "end": 1436 vals = line2.split(",") 1437 self.member_vars[var].append([]) 1438 if line2: 1439 for val in vals: 1440 self.member_vars[var][-1].append(float(val)) 1441 line2 = f.readline()[:-1] 1442 else: 1443 print("Did not find a member variable") 1444 1445 pdb.set_trace()
Load tracker values from string.
Parameters
- string (str): The string to load from.
Returns
- None: This function does not return a value.
def
from_string_debug(self, string):
1447 def from_string_debug(self, string): 1448 """Debug function to print tracker values from string without loading them. 1449 1450 Parameters 1451 ---------- 1452 string : str 1453 The string to debug load from. 1454 1455 Returns 1456 ------- 1457 None 1458 This function does not return a value. 1459 """ 1460 f = io.StringIO(string) 1461 print("=== DEBUGGING TRACKER VARIABLES ===") 1462 1463 while True: 1464 line = f.readline() 1465 if not line: 1466 break 1467 vals = line.split(",") 1468 var = vals[0] 1469 1470 print(f"\nVariable: {var}") 1471 print(f"Type: {self.member_var_types.get(var, 'UNKNOWN TYPE')}") 1472 print(f"Current value: {self.member_vars.get(var, 'NOT SET')}") 1473 1474 if self.member_var_types.get(var) == "bool": 1475 val = vals[1][:-1] 1476 print(f"Would set to: {val} -> {val == 'True'}") 1477 1478 elif self.member_var_types.get(var) == "int": 1479 val = vals[1] 1480 print(f"Would set to: {int(val)}") 1481 1482 elif self.member_var_types.get(var) == "float": 1483 val = vals[1] 1484 print(f"Would set to: {float(val)}") 1485 1486 elif self.member_var_types.get(var) == "string": 1487 val = vals[1][:-1] 1488 print(f"Would set to: '{val}'") 1489 1490 elif self.member_var_types.get(var) == "type": 1491 print("Would skip (type loading)") 1492 1493 elif self.member_var_types.get(var) == "empty array": 1494 val = vals[1] 1495 print(f"Would set to: [] (empty array)") 1496 1497 elif self.member_var_types.get(var) == "int array": 1498 vals_line = f.readline()[:-1].split(",") 1499 print(f"Would set to int array with {len(vals_line)} elements:") 1500 if vals_line[0] != "": 1501 print( 1502 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1503 ) 1504 else: 1505 print(" Empty array") 1506 1507 elif self.member_var_types.get(var) == "float array": 1508 vals_line = f.readline()[:-1].split(",") 1509 print(f"Would set to float array with {len(vals_line)} elements:") 1510 if vals_line[0] != "": 1511 print( 1512 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1513 ) 1514 else: 1515 print(" Empty array") 1516 1517 elif self.member_var_types.get(var) == "float array dictionary array": 1518 print("Would process float array dictionary array:") 1519 array_count = 0 1520 line2 = f.readline()[:-1] 1521 while line2 != "endarray": 1522 key_count = 0 1523 while line2 != "endkey": 1524 vals_dict = line2.split(",") 1525 name = vals_dict[0] 1526 print( 1527 f" Array {array_count}, Key '{name}': {len(vals_dict)-1} elements" 1528 ) 1529 key_count += 1 1530 line2 = f.readline()[:-1] 1531 print(f" Array {array_count} has {key_count} keys") 1532 array_count += 1 1533 line2 = f.readline()[:-1] 1534 print(f" Total arrays: {array_count}") 1535 1536 elif self.member_var_types.get(var) == "float array dictionary": 1537 print("Would process float array dictionary:") 1538 line2 = f.readline()[:-1] 1539 key_count = 0 1540 while line2 != "end": 1541 vals_dict = line2.split(",") 1542 name = vals_dict[0] 1543 print(f" Key '{name}': {len(vals_dict)-1} elements") 1544 key_count += 1 1545 line2 = f.readline()[:-1] 1546 print(f" Total keys: {key_count}") 1547 1548 elif self.member_var_types.get(var) == "float array array": 1549 print("Would process float array array:") 1550 line2 = f.readline()[:-1] 1551 array_count = 0 1552 while line2 != "end": 1553 if line2: 1554 vals_array = line2.split(",") 1555 print(f" Array {array_count}: {len(vals_array)} elements") 1556 else: 1557 print(f" Array {array_count}: empty") 1558 array_count += 1 1559 line2 = f.readline()[:-1] 1560 print(f" Total arrays: {array_count}") 1561 1562 else: 1563 print(f"UNKNOWN TYPE: {self.member_var_types.get(var, 'NOT FOUND')}") 1564 1565 print("\n=== END DEBUG ===")
Debug function to print tracker values from string without loading them.
Parameters
- string (str): The string to debug load from.
Returns
- None: This function does not return a value.
def
save_tracker_settings(self):
1567 def save_tracker_settings(self): 1568 """Save tracker settings for DistributedDataParallel use. 1569 1570 Saves settings in save_name/array_dims.csv 1571 1572 Parameters 1573 ---------- 1574 None 1575 Returns 1576 ------- 1577 None 1578 1579 ----- 1580 Instructions for use are in API customization.md 1581 """ 1582 if not os.path.isdir(self.save_name): 1583 os.makedirs(self.save_name) 1584 f = open(self.save_name + "/array_dims.csv", "w") 1585 for layer in self.neuron_module_vector: 1586 dv = layer.dendrite_module.dendrite_values[0] 1587 shape_str = ",".join(str(s) for s in dv.dendrite_storage_shape.tolist()) 1588 f.write(f"{layer.name},{shape_str}\n") 1589 f.close() 1590 if not GPA.pc.get_silent(): 1591 print("Tracker settings saved.") 1592 print("You may now delete save_tracker_settings")
Save tracker settings for DistributedDataParallel use.
Saves settings in save_name/array_dims.csv
Parameters
- None
Returns
- None
- -----
- Instructions for use are in API customization.md
def
initialize_tracker_settings(self):
1594 def initialize_tracker_settings(self): 1595 """Initialize tracker settings from saved file. 1596 1597 This function loads tracker settings from a CSV file and applies them 1598 to the layers the tracker is managing. 1599 1600 Parameters 1601 ---------- 1602 None 1603 1604 Returns 1605 ------- 1606 None 1607 1608 """ 1609 1610 channels = {} 1611 if not os.path.exists(self.save_name + "/array_dims.csv"): 1612 print( 1613 "You must call save_tracker_settings before " 1614 "initialize_tracker_settings" 1615 ) 1616 print("Follow instructions in customization.md") 1617 pdb.set_trace() 1618 f = open(self.save_name + "/array_dims.csv", "r") 1619 for line in f: 1620 parts = line.strip().split(",") 1621 channels[parts[0]] = [int(s) for s in parts[1:]] 1622 for layer in self.neuron_module_vector: 1623 dv = layer.dendrite_module.dendrite_values[0] 1624 dv.setup_arrays(channels[layer.name])
Initialize tracker settings from saved file.
This function loads tracker settings from a CSV file and applies them to the layers the tracker is managing.
Parameters
- None
Returns
- None
def
set_optimizer_instance(self, optimizer_instance, additional_optimizers=[]):
1626 def set_optimizer_instance(self, optimizer_instance, additional_optimizers=[]): 1627 """Set optimizer instance directly. 1628 1629 Parameters 1630 ---------- 1631 optimizer_instance : object 1632 The optimizer instance to set. 1633 1634 Returns 1635 ------- 1636 None 1637 1638 """ 1639 # This call must be first before the parameters get filtered. 1640 optimizer_instance.zero_grad() 1641 try: 1642 for param_group in optimizer_instance.param_groups: 1643 if ( 1644 param_group["weight_decay"] > 0 1645 and GPA.pc.get_weight_decay_accepted() is False 1646 ): 1647 _pai_log( 1648 "warning", 1649 "For PAI training it is recommended to not use weight decay in your optimizer", 1650 ) 1651 1652 except: 1653 pass 1654 self.member_vars["optimizer_instance"] = optimizer_instance 1655 if GPA.pc.get_perforated_backpropagation(): 1656 TPB.setup_optimizer_pb(self.member_vars["optimizer_instance"]) 1657 for optimizer in additional_optimizers: 1658 TPB.filter_params(optimizer)
Set optimizer instance directly.
Parameters
- optimizer_instance (object): The optimizer instance to set.
Returns
- None
def
set_optimizer(self, optimizer):
1660 def set_optimizer(self, optimizer): 1661 """Set optimizer type to be initialized later 1662 1663 Parameters 1664 ---------- 1665 optimizer : object 1666 The optimizer type to set. 1667 1668 Returns 1669 ------- 1670 None 1671 1672 """ 1673 self.member_vars["optimizer"] = optimizer
Set optimizer type to be initialized later
Parameters
- optimizer (object): The optimizer type to set.
Returns
- None
def
set_scheduler(self, scheduler):
1675 def set_scheduler(self, scheduler): 1676 """Set scheduler type to be initialized later 1677 1678 Parameters 1679 ---------- 1680 scheduler : object 1681 The scheduler type to set. 1682 1683 Returns 1684 ------- 1685 None 1686 1687 """ 1688 if scheduler is not torch.optim.lr_scheduler.ReduceLROnPlateau: 1689 if GPA.pc.get_verbose(): 1690 print("Not using ReduceLROnPlateau, this is not recommended") 1691 self.member_vars["scheduler"] = scheduler
Set scheduler type to be initialized later
Parameters
- scheduler (object): The scheduler type to set.
Returns
- None
def
increment_scheduler(self, num_ticks, mode):
1693 def increment_scheduler(self, num_ticks, mode): 1694 """Increment the scheduler a set number of times. 1695 1696 Used for finding best initial learning rate when adding dendrites. 1697 1698 Parameters 1699 ---------- 1700 num_ticks : int 1701 The number of scheduler steps to take. 1702 mode : str 1703 The mode for stepping the scheduler. Options are: 1704 - "step_learning_rate": Step based on improved accuracy epochs 1705 - "increment_epoch_count": Step based on total epoch count 1706 1707 Returns 1708 ------- 1709 current_steps : int 1710 The number of learning rate changes that occurred. 1711 learning_rate1 : float 1712 The final learning rate after stepping. 1713 1714 """ 1715 1716 current_steps = 0 1717 current_ticker = 0 1718 1719 for param_group in GPA.pai_tracker.member_vars[ 1720 "optimizer_instance" 1721 ].param_groups: 1722 learning_rate1 = param_group["lr"] 1723 1724 if GPA.pc.get_verbose(): 1725 print("Using scheduler:") 1726 print(type(self.member_vars["scheduler_instance"])) 1727 1728 while current_ticker < num_ticks: 1729 if GPA.pc.get_verbose(): 1730 print( 1731 f"Lower start rate initial {learning_rate1} " 1732 f'stepping {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} times' 1733 ) 1734 1735 if ( 1736 type(self.member_vars["scheduler_instance"]) 1737 is torch.optim.lr_scheduler.ReduceLROnPlateau 1738 ): 1739 if mode == "step_learning_rate": 1740 # Step with counter as last improved accuracy 1741 self.member_vars["scheduler_instance"].step( 1742 metrics=self.member_vars["last_improved_accuracies"][ 1743 GPA.pai_tracker.steps_after_switch() - 1 1744 ] 1745 ) 1746 elif mode == "increment_epoch_count": 1747 # Step with improved epoch counts up to current location 1748 self.member_vars["scheduler_instance"].step( 1749 metrics=self.member_vars["last_improved_accuracies"][ 1750 -((num_ticks - 1) - current_ticker) - 1 1751 ] 1752 ) 1753 else: 1754 self.member_vars["scheduler_instance"].step() 1755 1756 for param_group in GPA.pai_tracker.member_vars[ 1757 "optimizer_instance" 1758 ].param_groups: 1759 learning_rate2 = param_group["lr"] 1760 1761 if learning_rate2 != learning_rate1: 1762 current_steps += 1 1763 learning_rate1 = learning_rate2 1764 if mode == "step_learning_rate": 1765 current_ticker += 1 1766 if GPA.pc.get_verbose(): 1767 print(f"1 step {current_steps} to {learning_rate2}") 1768 1769 if mode == "increment_epoch_count": 1770 current_ticker += 1 1771 1772 return current_steps, learning_rate1
Increment the scheduler a set number of times.
Used for finding best initial learning rate when adding dendrites.
Parameters
- num_ticks (int): The number of scheduler steps to take.
- mode (str):
The mode for stepping the scheduler. Options are:
- "step_learning_rate": Step based on improved accuracy epochs
- "increment_epoch_count": Step based on total epoch count
Returns
- current_steps (int): The number of learning rate changes that occurred.
- learning_rate1 (float): The final learning rate after stepping.
def
setup_optimizer(self, net, opt_args, sched_args=None, parameters=None):
1774 def setup_optimizer(self, net, opt_args, sched_args=None, parameters=None): 1775 """Initialize the optimizer and scheduler when added. 1776 1777 Parameters 1778 ---------- 1779 net : object 1780 The neural network model. 1781 opt_args : dict 1782 The arguments for the optimizer. 1783 sched_args : dict, optional 1784 The arguments for the scheduler, by default None. 1785 1786 Returns 1787 ------- 1788 optimizer : object 1789 The initialized optimizer instance. 1790 scheduler : object or None 1791 The initialized scheduler instance, or None if no scheduler was set. 1792 1793 """ 1794 if "weight_decay" in opt_args and not GPA.pc.get_weight_decay_accepted(): 1795 _pai_log( 1796 "warning", 1797 "For PAI training it is recommended to not use weight decay in your optimizer", 1798 ) 1799 1800 if ("model" not in opt_args.keys()) and "params" not in opt_args.keys(): 1801 print("In setup_optimizer it will be depreciated to not pass in params yourself in the future") 1802 print("please change the settings to include params") 1803 if self.member_vars["mode"] == "n": 1804 if parameters is not None: 1805 opt_args["params"] = parameters 1806 else: 1807 opt_args["params"] = filter(lambda p: p.requires_grad, net.parameters()) 1808 else: 1809 params = UPA.get_pai_network_params(net) 1810 if parameters is not None: 1811 # Filter parameters to only those in params, preserving weight_decay 1812 params_set = set(params) 1813 filtered_params = [] 1814 for param_group in parameters: 1815 filtered_group_params = [p for p in param_group["params"] if p in params_set] 1816 if filtered_group_params: 1817 filtered_params.append({ 1818 "params": filtered_group_params, 1819 "weight_decay": param_group["weight_decay"] 1820 }) 1821 opt_args["params"] = filtered_params 1822 else: 1823 opt_args["params"] = params 1824 elif "params" in opt_args.keys(): 1825 # Check if params is a list of param groups (dicts) or a single param group 1826 params_value = opt_args["params"] 1827 if isinstance(params_value, list) and len(params_value) > 0: 1828 # Check if it's a list of dicts (multiple param groups) or list of tensors (single group) 1829 if isinstance(params_value[0], dict): 1830 # Multiple param groups format: [{"params": [...], "lr": ...}, ...] 1831 # Filter each param group for requires_grad 1832 filtered_param_groups = [] 1833 for param_group in params_value: 1834 filtered_group_params = [p for p in param_group["params"] if p.requires_grad] 1835 if filtered_group_params: 1836 new_group = param_group.copy() 1837 new_group["params"] = filtered_group_params 1838 filtered_param_groups.append(new_group) 1839 opt_args["params"] = filtered_param_groups 1840 else: 1841 # Single param group format: [tensor1, tensor2, ...] or generator 1842 # Filter for requires_grad 1843 opt_args["params"] = [p for p in params_value if p.requires_grad] 1844 elif hasattr(params_value, '__iter__'): 1845 # Handle generators or other iterables 1846 opt_args["params"] = [p for p in params_value if p.requires_grad] 1847 1848 optimizer = self.member_vars["optimizer"](**opt_args) 1849 self.set_optimizer_instance(optimizer) 1850 1851 if self.member_vars["scheduler"] is not None: 1852 # Handle SequentialLR specially 1853 if self.member_vars["scheduler"] is torch.optim.lr_scheduler.SequentialLR: 1854 """ 1855 sched_args should be a dict with "schedulers" (list of tuples) and "milestones" 1856 For example: 1857 sequential_schedArgs = { 1858 "schedulers": [ 1859 (warmup_scheduler_class, warmup_schedArgs), 1860 (main_scheduler_class, main_schedArgs) 1861 ], 1862 "milestones": [switch_epoch] 1863 } 1864 """ 1865 schedulers = [] 1866 milestones = sched_args.get("milestones", []) 1867 scheduler_configs = sched_args.get("schedulers", []) 1868 1869 for scheduler_class, scheduler_args in scheduler_configs: 1870 schedulers.append(scheduler_class(optimizer, **scheduler_args)) 1871 1872 self.member_vars["scheduler_instance"] = torch.optim.lr_scheduler.SequentialLR( 1873 optimizer, schedulers=schedulers, milestones=milestones 1874 ) 1875 else: 1876 self.member_vars["scheduler_instance"] = self.member_vars["scheduler"]( 1877 optimizer, **sched_args 1878 ) 1879 current_steps = 0 1880 1881 for param_group in GPA.pai_tracker.member_vars[ 1882 "optimizer_instance" 1883 ].param_groups: 1884 learning_rate1 = param_group["lr"] 1885 1886 if GPA.pc.get_verbose(): 1887 print( 1888 f"Resetting scheduler with {GPA.pai_tracker.steps_after_switch()} " 1889 f"steps and {GPA.pc.get_initial_history_after_switches()} initial ticks to skip" 1890 ) 1891 1892 # Find setting of previously used learning rate before adding dendrites 1893 if ( 1894 GPA.pai_tracker.member_vars[ 1895 "current_n_learning_rate_initial_skip_steps" 1896 ] 1897 != 0 1898 ): 1899 additional_steps, learning_rate1 = self.increment_scheduler( 1900 GPA.pai_tracker.member_vars[ 1901 "current_n_learning_rate_initial_skip_steps" 1902 ], 1903 "step_learning_rate", 1904 ) 1905 current_steps += additional_steps 1906 1907 if self.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 1908 initial = GPA.pc.get_initial_history_after_switches() 1909 else: 1910 initial = 0 1911 1912 if GPA.pai_tracker.steps_after_switch() > initial: 1913 # Minus extra 1 because this gets called after start epoch 1914 additional_steps, learning_rate1 = self.increment_scheduler( 1915 (GPA.pai_tracker.steps_after_switch() - initial) - 1, 1916 "increment_epoch_count", 1917 ) 1918 current_steps += additional_steps 1919 1920 if GPA.pc.get_verbose(): 1921 print( 1922 f"Scheduler update loop with {current_steps} " 1923 f"ended with {learning_rate1}" 1924 ) 1925 print( 1926 f"Scheduler ended with {current_steps} steps " 1927 f"and lr of {learning_rate1}" 1928 ) 1929 1930 self.member_vars["current_step_count"] = current_steps 1931 return optimizer, self.member_vars["scheduler_instance"] 1932 else: 1933 return optimizer, None
Initialize the optimizer and scheduler when added.
Parameters
- net (object): The neural network model.
- opt_args (dict): The arguments for the optimizer.
- sched_args (dict, optional): The arguments for the scheduler, by default None.
Returns
- optimizer (object): The initialized optimizer instance.
- scheduler (object or None): The initialized scheduler instance, or None if no scheduler was set.
def
clear_optimizer_and_scheduler(self):
1935 def clear_optimizer_and_scheduler(self): 1936 """Clear the instances for saving. 1937 1938 Parameters 1939 ---------- 1940 None 1941 1942 Returns 1943 ------- 1944 None 1945 This function does not return a value. 1946 """ 1947 self.member_vars["optimizer_instance"] = None 1948 self.member_vars["scheduler_instance"] = None
Clear the instances for saving.
Parameters
- None
Returns
- None: This function does not return a value.
def
switch_time(self):
1950 def switch_time(self): 1951 """Determine if it's time to switch between neuron and dendrite training. 1952 1953 Parameters 1954 ---------- 1955 None 1956 1957 Returns 1958 ------- 1959 bool 1960 True if it's time to switch, False otherwise. 1961 1962 Notes 1963 ----- 1964 Based on current settings and history of scores. 1965 """ 1966 1967 switch_phrase = "No mode, this should never be the case." 1968 switch_number = GPA.pc.get_n_epochs_to_switch() 1969 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1970 switch_phrase = "DOING_SWITCH_EVERY_TIME" 1971 elif self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY: 1972 switch_phrase = "DOING_HISTORY" 1973 elif self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH: 1974 switch_phrase = "DOING_FIXED_SWITCH" 1975 switch_number = GPA.pc.get_fixed_switch_num() 1976 elif self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1977 switch_phrase = "DOING_NO_SWITCH" 1978 else: 1979 print( 1980 "A switch mode must be set. Check your settings for GPA.pc.set_switch_mode()." 1981 ) 1982 pdb.set_trace() 1983 if not GPA.pc.get_silent(): 1984 if(GPA.pc.get_perforated_backpropagation()): 1985 print( 1986 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1987 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1988 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1989 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1990 f'n: {switch_number}, p: {GPA.pc.get_p_epochs_to_switch()}, ' 1991 f'num_cycles: {self.member_vars["num_cycles"]}' 1992 ) 1993 else: 1994 print( 1995 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1996 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1997 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1998 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1999 f'n: {switch_number}, num_cycles: {self.member_vars["num_cycles"]}' 2000 ) 2001 print( 2002 f' Score tracking: current_n_set_global_best={self.member_vars["current_n_set_global_best"]}, ' 2003 f'global_best={self.member_vars["global_best_validation_score"]:.4f}, ' 2004 f'current_best={self.member_vars["current_best_validation_score"]:.4f}' 2005 ) 2006 if GPA.pc.get_perforated_backpropagation(): 2007 # this will fill in epoch last improved 2008 TPB.best_pai_score_improved_this_epoch(self) ## CLOSED ONLY 2009 if self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 2010 if not GPA.pc.get_silent(): 2011 print("Returning False - doing no switch mode") 2012 return False 2013 2014 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 2015 if not GPA.pc.get_silent(): 2016 print("Returning True - switching every time") 2017 return True 2018 2019 # Check if we're in the middle of learning rate optimization 2020 # If so, block ALL switch triggers until committed 2021 if GPA.pc.get_verbose(): 2022 print("=== LR Optimization Check ===") 2023 print(f' mode == "n": {self.member_vars["mode"] == "n"}') 2024 print(f" get_learn_dendrites_live(): {GPA.pc.get_learn_dendrites_live()}") 2025 print(f' committed_to_initial_rate: {GPA.pai_tracker.member_vars["committed_to_initial_rate"]}') 2026 print(f" get_dont_give_up_unless_learning_rate_lowered(): {GPA.pc.get_dont_give_up_unless_learning_rate_lowered()}") 2027 print(f' current_n_learning_rate_initial_skip_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"]}') 2028 print(f' last_max_learning_rate_steps: {self.member_vars["last_max_learning_rate_steps"]}') 2029 print(f' skip_steps < max_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"] < self.member_vars["last_max_learning_rate_steps"]}') 2030 print(f' scheduler is not None: {self.member_vars["scheduler"] is not None}') 2031 print("=============================") 2032 2033 if ( 2034 ((self.member_vars["mode"] == "n") or GPA.pc.get_learn_dendrites_live()) 2035 and (GPA.pai_tracker.member_vars["committed_to_initial_rate"] is False) 2036 and (GPA.pc.get_dont_give_up_unless_learning_rate_lowered()) 2037 and ( 2038 self.member_vars["current_n_learning_rate_initial_skip_steps"] 2039 <= self.member_vars["last_max_learning_rate_steps"] 2040 ) 2041 and self.member_vars["scheduler"] is not None 2042 ): 2043 if not GPA.pc.get_silent(): 2044 print( 2045 f"Returning False - learning rate optimization in progress. " 2046 f"Not committed yet. Comparing " 2047 f'initial {self.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 2048 f'to last max {self.member_vars["last_max_learning_rate_steps"]}' 2049 ) 2050 return False 2051 2052 if len(self.member_vars["switch_epochs"]) == 0: 2053 this_count = self.member_vars["num_epochs_run"] 2054 else: 2055 this_count = ( 2056 self.member_vars["num_epochs_run"] 2057 - self.member_vars["switch_epochs"][-1] 2058 ) 2059 cap_switch = False 2060 if GPA.pc.get_perforated_backpropagation(): 2061 cap_switch = TPB.check_cap_switch(self, this_count) 2062 2063 if self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY and ( 2064 ( 2065 (self.member_vars["mode"] == "n") 2066 and ( 2067 self.member_vars["num_epochs_run"] 2068 - self.member_vars["epoch_last_improved"] 2069 >= GPA.pc.get_n_epochs_to_switch() 2070 ) 2071 and this_count 2072 >= GPA.pc.get_initial_history_after_switches() 2073 + GPA.pc.get_n_epochs_to_switch() 2074 ) 2075 or (GPA.pc.get_perforated_backpropagation() and TPB.history_switch(self)) 2076 or cap_switch 2077 ): 2078 if not GPA.pc.get_silent(): 2079 print("Returning True - History and last improved is hit") 2080 return True 2081 2082 if self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH and ( 2083 ( 2084 self.member_vars["total_epochs_run"] % GPA.pc.get_fixed_switch_num() 2085 == GPA.pc.get_fixed_switch_num() - 1 2086 ) 2087 and self.member_vars["num_epochs_run"] 2088 >= GPA.pc.get_first_fixed_switch_num() - 1 2089 ): 2090 if not GPA.pc.get_silent(): 2091 print("Returning True - Fixed switch number is hit") 2092 return True 2093 2094 if not GPA.pc.get_silent(): 2095 print("Returning False - no triggers to switch have been hit") 2096 return False
Determine if it's time to switch between neuron and dendrite training.
Parameters
- None
Returns
- bool: True if it's time to switch, False otherwise.
Notes
Based on current settings and history of scores.
def
steps_after_switch(self):
2098 def steps_after_switch(self): 2099 """Based on settings, return value for steps since a switch. 2100 2101 Different options for param vals setting determine what is returned. 2102 2103 Parameters 2104 ---------- 2105 None 2106 2107 Returns 2108 ------- 2109 int 2110 The number of epochs since the last switch, or total epochs run, 2111 depending on settings. 2112 2113 """ 2114 if self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_TOTAL_EPOCH: 2115 return self.member_vars["num_epochs_run"] 2116 elif ( 2117 self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_UPDATE_EPOCH 2118 ): 2119 return self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2120 elif ( 2121 self.member_vars["param_vals_setting"] 2122 == GPA.pc.PARAM_VALS_BY_NEURON_EPOCH_START 2123 ): 2124 if self.member_vars["mode"] == "p": 2125 return ( 2126 self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2127 ) 2128 else: 2129 return self.member_vars["num_epochs_run"] 2130 else: 2131 print( 2132 f'{self.member_vars["param_vals_setting"]} is not a valid param vals option' 2133 ) 2134 pdb.set_trace()
Based on settings, return value for steps since a switch.
Different options for param vals setting determine what is returned.
Parameters
- None
Returns
- int: The number of epochs since the last switch, or total epochs run, depending on settings.
def
add_pai_neuron_module(self, new_module, initial_add=True):
2136 def add_pai_neuron_module(self, new_module, initial_add=True): 2137 """Add neuron modules to internal vectors. 2138 2139 Parameters 2140 ---------- 2141 new_module : object 2142 The new module to add. 2143 initial_add : bool, optional 2144 Whether this is the initial addition rather than loading from file 2145 2146 Returns 2147 ------- 2148 None 2149 2150 """ 2151 2152 # If it's a duplicate, ignore the second addition 2153 if new_module in self.neuron_module_vector: 2154 return 2155 self.neuron_module_vector.append(new_module) 2156 if self.member_vars["doing_pai"]: 2157 PA.set_wrapped_params(new_module) 2158 if initial_add: 2159 self.member_vars["best_scores"].append([]) 2160 self.member_vars["current_scores"].append([])
Add neuron modules to internal vectors.
Parameters
- new_module (object): The new module to add.
- initial_add (bool, optional): Whether this is the initial addition rather than loading from file
Returns
- None
def
add_tracked_neuron_module(self, new_module, initial_add=True):
2162 def add_tracked_neuron_module(self, new_module, initial_add=True): 2163 """Add tracked modules to internal vectors 2164 2165 Parameters 2166 ---------- 2167 new_module : object 2168 The new module to add. 2169 initial_add : bool, optional 2170 Whether this is the initial addition rather than loading from file 2171 2172 Returns 2173 ------- 2174 None 2175 2176 """ 2177 # If it's a duplicate, ignore the second addition 2178 if new_module in self.tracked_neuron_module_vector: 2179 return 2180 self.tracked_neuron_module_vector.append(new_module) 2181 if self.member_vars["doing_pai"]: 2182 PA.set_tracked_params(new_module)
Add tracked modules to internal vectors
Parameters
- new_module (object): The new module to add.
- initial_add (bool, optional): Whether this is the initial addition rather than loading from file
Returns
- None
def
reset_module_vector(self, net, load_from_restart):
2184 def reset_module_vector(self, net, load_from_restart): 2185 """Clear internal vectors and reset from network. 2186 2187 Parameters 2188 ---------- 2189 net : object 2190 The neural network model. 2191 load_from_restart : bool 2192 Whether loading from a restart file. 2193 2194 Returns 2195 ------- 2196 None 2197 2198 """ 2199 self.neuron_module_vector = [] 2200 self.tracked_neuron_module_vector = [] 2201 this_list = UPA.get_pai_modules(net, 0) 2202 for module in this_list: 2203 self.add_pai_neuron_module(module, initial_add=load_from_restart) 2204 this_list = UPA.get_tracked_modules(net, 0) 2205 for module in this_list: 2206 self.add_tracked_neuron_module(module, initial_add=load_from_restart)
Clear internal vectors and reset from network.
Parameters
- net (object): The neural network model.
- load_from_restart (bool): Whether loading from a restart file.
Returns
- None
def
reset_vals_for_score_reset(self):
2208 def reset_vals_for_score_reset(self): 2209 """Reset cycle scores for new cycle. 2210 2211 Parameters 2212 ---------- 2213 None 2214 2215 Returns 2216 ------- 2217 None 2218 This function does not return a value. 2219 """ 2220 2221 if GPA.pc.get_find_best_lr(): 2222 self.member_vars["committed_to_initial_rate"] = False 2223 print("Resetting committed to initial rate to False") 2224 # If retaining all dendrties always say that the current dendrites set global best for saving and loading 2225 if GPA.pc.get_retain_all_dendrites(): 2226 self.member_vars["current_n_set_global_best"] = True 2227 self.member_vars["global_best_validation_score"] = 0 2228 else: 2229 self.member_vars["current_n_set_global_best"] = False 2230 2231 # Don't reset global best, but do reset current best 2232 self.member_vars["current_best_validation_score"] = 0 2233 self.member_vars["initial_lr_test_epoch_count"] = -1
Reset cycle scores for new cycle.
Parameters
- None
Returns
- None: This function does not return a value.
def
set_dendrite_training(self):
2235 def set_dendrite_training(self): 2236 """Signal all layers to start dendrite training. 2237 2238 Parameters 2239 ---------- 2240 None 2241 2242 Returns 2243 ------- 2244 None 2245 This function does not return a value. 2246 """ 2247 if GPA.pc.get_verbose(): 2248 print("Calling set_dendrite_training") 2249 2250 for layer in self.neuron_module_vector[:]: 2251 worked = layer.set_mode("p") 2252 """ 2253 worked is False when a layer was added to the neuron module vector 2254 but then it's never actually been used. This can happen when 2255 you have set a layer to have requires_grad = False or when 2256 you have a module as a member variable but it's not actually 2257 part of the network. Should be moved to be a tracked layer 2258 rather than a neuron layer. 2259 """ 2260 if not worked: 2261 self.neuron_module_vector.remove(layer) 2262 2263 for layer in self.tracked_neuron_module_vector[:]: 2264 worked = layer.set_mode("p") 2265 2266 self.create_new_dendrite_module() 2267 self.member_vars["mode"] = "p" 2268 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2269 2270 if GPA.pc.get_learn_dendrites_live(): 2271 self.reset_vals_for_score_reset() 2272 2273 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2274 "current_step_count" 2275 ] 2276 2277 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 2278 GPA.pai_tracker.member_vars["num_cycles"] += 1
Signal all layers to start dendrite training.
Parameters
- None
Returns
- None: This function does not return a value.
def
set_neuron_training(self):
2281 def set_neuron_training(self): 2282 """Signal all layers to start neuron training. 2283 2284 Parameters 2285 ---------- 2286 None 2287 2288 Returns 2289 ------- 2290 None 2291 This function does not return a value. 2292 """ 2293 for module in self.neuron_module_vector: 2294 module.set_mode("n") 2295 for module in self.tracked_neuron_module_vector[:]: 2296 module.set_mode("n") 2297 2298 self.member_vars["mode"] = "n" 2299 self.member_vars["num_dendrites_added"] += 1 2300 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2301 self.reset_vals_for_score_reset() 2302 2303 self.member_vars["current_cycle_lr_max_scores"] = [] 2304 if GPA.pc.get_learn_dendrites_live(): 2305 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2306 "current_step_count" 2307 ] 2308 GPA.pai_tracker.member_vars["num_cycles"] += 1 2309 2310 if GPA.pc.get_reset_best_score_on_switch(): 2311 GPA.pai_tracker.member_vars["current_best_validation_score"] = 0 2312 GPA.pai_tracker.member_vars["running_accuracy"] = 0
Signal all layers to start neuron training.
Parameters
- None
Returns
- None: This function does not return a value.
def
start_epoch(self, internal_call=False):
2314 def start_epoch(self, internal_call=False): 2315 """Perform steps for when a new training epoch is about to begin. 2316 2317 Parameters 2318 ---------- 2319 internal_call : bool, optional 2320 Whether this is an internal call or manual call 2321 2322 Returns 2323 ------- 2324 None 2325 2326 Notes 2327 ----- 2328 If you ever need to call this manually, set internal_call to False. 2329 2330 """ 2331 if self.member_vars["manual_train_switch"] and internal_call: 2332 return 2333 2334 if not internal_call and not self.member_vars["manual_train_switch"]: 2335 self.member_vars["manual_train_switch"] = True 2336 self.saved_time = 0 2337 self.member_vars["num_epochs_run"] = -1 2338 self.member_vars["total_epochs_run"] = -1 2339 2340 end = time.time() 2341 if self.member_vars["manual_train_switch"]: 2342 if self.saved_time != 0: 2343 if self.member_vars["mode"] == "p": 2344 self.member_vars["p_val_times"].append(end - self.saved_time) 2345 else: 2346 self.member_vars["n_val_times"].append(end - self.saved_time) 2347 2348 if self.member_vars["mode"] == "p": 2349 for layer in self.neuron_module_vector: 2350 for m in range(0, GPA.pc.get_global_candidates()): 2351 with torch.no_grad(): 2352 if GPA.pc.get_verbose(): 2353 print(f"Resetting score for {layer.name}") 2354 # Snapshot best_score before reset so we can compute per-epoch improvement 2355 layer.dendrite_module.dendrite_values[ 2356 m 2357 ].epoch_start_best_score.copy_( 2358 layer.dendrite_module.dendrite_values[ 2359 m 2360 ].best_score.detach() 2361 ) 2362 layer.dendrite_module.dendrite_values[ 2363 m 2364 ].best_score_improved_this_epoch = ( 2365 layer.dendrite_module.dendrite_values[ 2366 m 2367 ].best_score_improved_this_epoch 2368 * 0 2369 ) 2370 layer.dendrite_module.dendrite_values[ 2371 m 2372 ].nodes_best_improved_this_epoch = ( 2373 layer.dendrite_module.dendrite_values[ 2374 m 2375 ].nodes_best_improved_this_epoch 2376 * 0 2377 ) 2378 layer.dendrite_module.dendrite_values[ 2379 m 2380 ].nodes_improved_any = ( 2381 layer.dendrite_module.dendrite_values[ 2382 m 2383 ].nodes_improved_any 2384 * 0 2385 ) 2386 if GPA.pc.get_perforated_backpropagation(): 2387 self.member_vars["best_mean_score_improved_this_epoch"] = 0 2388 self.member_vars["num_epochs_run"] += 1 2389 self.member_vars["total_epochs_run"] = ( 2390 self.member_vars["num_epochs_run"] + self.member_vars["overwritten_epochs"] 2391 ) 2392 self.saved_time = end
Perform steps for when a new training epoch is about to begin.
Parameters
- internal_call (bool, optional): Whether this is an internal call or manual call
Returns
- None
Notes
If you ever need to call this manually, set internal_call to False.
def
stop_epoch(self, internal_call=False):
2394 def stop_epoch(self, internal_call=False): 2395 """Perform steps when a training epoch has completed. 2396 2397 Parameters 2398 ---------- 2399 internal_call : bool, optional 2400 Whether this is an internal call or manual call 2401 2402 Returns 2403 ------- 2404 None 2405 2406 Notes 2407 ----- 2408 If you ever need to call this manually, set internal_call to False. 2409 2410 """ 2411 end = time.time() 2412 if self.member_vars["manual_train_switch"] and internal_call: 2413 return 2414 2415 if self.member_vars["manual_train_switch"]: 2416 if self.member_vars["mode"] == "p": 2417 self.member_vars["p_train_times"].append(end - self.saved_time) 2418 else: 2419 self.member_vars["n_train_times"].append(end - self.saved_time) 2420 else: 2421 if self.member_vars["mode"] == "p": 2422 self.member_vars["p_epoch_times"].append(end - self.saved_time) 2423 else: 2424 self.member_vars["n_epoch_times"].append(end - self.saved_time) 2425 2426 self.saved_time = end
Perform steps when a training epoch has completed.
Parameters
- internal_call (bool, optional): Whether this is an internal call or manual call
Returns
- None
Notes
If you ever need to call this manually, set internal_call to False.
def
initialize( self, model, doing_pai=True, save_name='PAI', making_graphs=True, maximizing_score=True, num_classes=10000, values_per_train_epoch=-1, values_per_val_epoch=-1, zooming_graph=True):
2428 def initialize( 2429 self, 2430 model, 2431 doing_pai=True, 2432 save_name="PAI", 2433 making_graphs=True, 2434 maximizing_score=True, 2435 num_classes=10000, 2436 values_per_train_epoch=-1, 2437 values_per_val_epoch=-1, 2438 zooming_graph=True, 2439 ): 2440 """Setup the tracker with initial settings. 2441 2442 2443 Parameters 2444 ---------- 2445 model : object 2446 The neural network model. 2447 doing_pai : bool, optional 2448 Whether to add dendrites, by default True. 2449 save_name : str, optional 2450 The name under which to save the model. 2451 making_graphs : bool, optional 2452 Whether to make graphs, by default True. 2453 maximizing_score : bool, optional 2454 Whether to maximize the score, by default True. 2455 num_classes : int, optional 2456 The number of classes in the dataset, unused 2457 values_per_train_epoch : int, optional 2458 The number of values to look back for graphing 2459 during training, by default -1 (all values). 2460 values_per_val_epoch : int, optional 2461 The number of values to look back for graphing 2462 during validation, by default -1 (all values). 2463 zooming_graph : bool, optional 2464 Whether to zoom on graphs, by default True. 2465 2466 2467 Returns 2468 ------- 2469 nn.Module 2470 Converted model instance configured for the tracker settings. 2471 """ 2472 model = UPA.convert_network(model) 2473 self.member_vars["doing_pai"] = doing_pai 2474 self.member_vars["maximizing_score"] = maximizing_score 2475 self.save_name = save_name 2476 self.zooming_graph = zooming_graph 2477 self.making_graphs = making_graphs 2478 2479 if not self.loaded: 2480 self.member_vars["running_accuracy"] = (1.0 / num_classes) * 100 2481 2482 self.values_per_train_epoch = values_per_train_epoch 2483 self.values_per_val_epoch = values_per_val_epoch 2484 2485 if GPA.pc.get_testing_dendrite_capacity(): 2486 if not GPA.pc.get_silent(): 2487 print("Running a test of Dendrite Capacity.") 2488 GPA.pc.set_switch_mode(GPA.pc.DOING_SWITCH_EVERY_TIME) 2489 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 2490 GPA.pc.set_retain_all_dendrites(True) 2491 GPA.pc.set_max_dendrite_tries(1000) 2492 GPA.pc.set_max_dendrites(1000) 2493 if GPA.pc.get_perforated_backpropagation(): 2494 GPA.pc.set_initial_correlation_batches(1) 2495 else: 2496 if not GPA.pc.get_silent(): 2497 print("Running Dendrite Experiment") 2498 return model
Setup the tracker with initial settings.
Parameters
- model (object): The neural network model.
- doing_pai (bool, optional): Whether to add dendrites, by default True.
- save_name (str, optional): The name under which to save the model.
- making_graphs (bool, optional): Whether to make graphs, by default True.
- maximizing_score (bool, optional): Whether to maximize the score, by default True.
- num_classes (int, optional): The number of classes in the dataset, unused
- values_per_train_epoch (int, optional): The number of values to look back for graphing during training, by default -1 (all values).
- values_per_val_epoch (int, optional): The number of values to look back for graphing during validation, by default -1 (all values).
- zooming_graph (bool, optional): Whether to zoom on graphs, by default True.
Returns
- nn.Module: Converted model instance configured for the tracker settings.
def
generate_accuracy_plots(self, ax, save_folder, extra_string):
2500 def generate_accuracy_plots(self, ax, save_folder, extra_string): 2501 """ 2502 Generate plots and csvs for accuracy 2503 2504 Parameters 2505 ---------- 2506 ax : object 2507 The matplotlib axis to plot on. 2508 save_folder : str 2509 The folder to save the plots and csvs in. 2510 extra_string : str 2511 An extra string to append to the filenames. 2512 2513 Returns 2514 ------- 2515 None 2516 2517 """ 2518 2519 # If scores are being saved for epochs that get overwritten, plot them 2520 for list_id in range(len(self.member_vars["overwritten_extras"])): 2521 for extra_id in self.member_vars["overwritten_extras"][list_id]: 2522 ax.plot( 2523 np.arange( 2524 len(self.member_vars["overwritten_extras"][list_id][extra_id]) 2525 ), 2526 self.member_vars["overwritten_extras"][list_id][extra_id], 2527 color=_PAI_PALETTE["train"], 2528 alpha=0.35, 2529 ) 2530 ax.plot( 2531 np.arange(len(self.member_vars["overwritten_vals"][list_id])), 2532 self.member_vars["overwritten_vals"][list_id], 2533 color=_PAI_PALETTE["validation"], 2534 alpha=0.35, 2535 ) 2536 2537 # Determine which accuracy vector to use 2538 if GPA.pc.get_drawing_pai(): 2539 accuracies = self.member_vars["accuracies"] 2540 else: 2541 accuracies = self.member_vars["n_accuracies"] 2542 2543 # Get pointer to additional scores being saved 2544 extra_scores = self.member_vars["extra_scores"] 2545 2546 # Plot the main accuracy scores 2547 ax.plot( 2548 np.arange(len(accuracies)), 2549 accuracies, 2550 label="Validation Scores", 2551 color=_PAI_PALETTE["validation"], 2552 ) 2553 ax.plot( 2554 np.arange(len(self.member_vars["running_accuracies"])), 2555 self.member_vars["running_accuracies"], 2556 label="Validation Running Scores", 2557 color=_PAI_PALETTE["validation"], 2558 alpha=0.4, 2559 ) 2560 2561 # Plot additional scores 2562 for extra_index, extra_score in enumerate(extra_scores): 2563 ax.plot( 2564 np.arange(len(extra_scores[extra_score])), 2565 extra_scores[extra_score], 2566 label=extra_score, 2567 color=_pai_extra_score_color(extra_score, extra_index), 2568 ) 2569 2570 plt.title(save_folder + "/" + self.save_name + "Scores") 2571 plt.xlabel("Epochs") 2572 plt.ylabel("Score") 2573 2574 # Add point at epoch last improved and best validation score 2575 if GPA.pc.get_drawing_pai(): 2576 ax.plot( 2577 self.member_vars["epoch_last_improved"], 2578 self.member_vars["global_best_validation_score"], 2579 "o", 2580 color=_PAI_PALETTE["marker_best"], 2581 label="Global best (y)", 2582 ) 2583 ax.plot( 2584 self.member_vars["epoch_last_improved"], 2585 accuracies[self.member_vars["epoch_last_improved"]], 2586 "o", 2587 color=_PAI_PALETTE["marker_improved"], 2588 label="Epoch Last Improved", 2589 ) 2590 else: 2591 if self.member_vars["mode"] == "n": 2592 missed_time = ( 2593 self.member_vars["num_epochs_run"] 2594 - self.member_vars["epoch_last_improved"] 2595 ) 2596 ax.plot( 2597 (len(self.member_vars["n_accuracies"]) - 1) - missed_time, 2598 self.member_vars["n_accuracies"][-(missed_time + 1)], 2599 "o", 2600 color=_PAI_PALETTE["marker_improved"], 2601 label="Epoch Last Improved", 2602 ) 2603 2604 # Generate csv file for the values graphed 2605 pd1 = pd.DataFrame( 2606 {"Epochs": np.arange(len(accuracies)), "Validation Scores": accuracies} 2607 ) 2608 pd2 = pd.DataFrame( 2609 { 2610 "Epochs": np.arange(len(self.member_vars["running_accuracies"])), 2611 "Validation Running Scores": self.member_vars["running_accuracies"], 2612 } 2613 ) 2614 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2615 for extra_score in extra_scores: 2616 pd2 = pd.DataFrame( 2617 { 2618 "Epochs": np.arange(len(extra_scores[extra_score])), 2619 extra_score: extra_scores[extra_score], 2620 } 2621 ) 2622 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2623 extra_scores_without_graphing = self.member_vars[ 2624 "extra_scores_without_graphing" 2625 ] 2626 for extra_score in extra_scores_without_graphing: 2627 pd2 = pd.DataFrame( 2628 { 2629 "Epochs": np.arange( 2630 len(extra_scores_without_graphing[extra_score]) 2631 ), 2632 extra_score: extra_scores_without_graphing[extra_score], 2633 } 2634 ) 2635 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2636 pd1.to_csv( 2637 save_folder + "/" + self.save_name + extra_string + "Scores.csv", 2638 index=False, 2639 ) 2640 del pd1, pd2 2641 2642 # Set y min and max to zoom in on important part of axis 2643 if ( 2644 len(self.member_vars["switch_epochs"]) > 0 2645 and self.member_vars["switch_epochs"][0] > 0 2646 and self.zooming_graph 2647 ): 2648 if GPA.pai_tracker.member_vars["maximizing_score"]: 2649 min_val = np.array( 2650 accuracies[0 : self.member_vars["switch_epochs"][0]] 2651 ).mean() 2652 for extra_score in extra_scores: 2653 min_pot = np.array( 2654 extra_scores[extra_score][ 2655 0 : self.member_vars["switch_epochs"][0] 2656 ] 2657 ).mean() 2658 if min_pot < min_val: 2659 min_val = min_pot 2660 ax.set_ylim(ymin=min_val) 2661 else: 2662 max_val = np.array( 2663 accuracies[0 : self.member_vars["switch_epochs"][0]] 2664 ).mean() 2665 for extra_score in extra_scores: 2666 max_pot = np.array( 2667 extra_scores[extra_score][ 2668 0 : self.member_vars["switch_epochs"][0] 2669 ] 2670 ).mean() 2671 if max_pot > max_val: 2672 max_val = max_pot 2673 ax.set_ylim(ymax=max_val) 2674 2675 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2676 _pai_grid(ax) 2677 2678 # Draw vertical lines for epochs where a dendrite switch occurred. 2679 # Colour by the phase being entered: dendrite/PAI phase -> teal, 2680 # normal phase -> orange, alternating starting with dendrite. 2681 if GPA.pc.get_drawing_pai() and self.member_vars["doing_pai"]: 2682 for i, switcher in enumerate(self.member_vars["switch_epochs"]): 2683 # Mode always starts at "n" and switch_epochs alternates 2684 # n->p->n->p..., so the first switch enters the dendrite 2685 # phase and every other one alternates from there. 2686 entering_p = (i % 2) == 0 2687 plt.axvline( 2688 x=switcher, 2689 ymin=0, 2690 ymax=1, 2691 color=_PAI_PALETTE["dendrite"] 2692 if entering_p 2693 else _PAI_PALETTE["normal"], 2694 ) 2695 else: 2696 for switcher in self.member_vars["n_switch_epochs"]: 2697 plt.axvline( 2698 x=switcher, ymin=0, ymax=1, color=_PAI_PALETTE["normal"] 2699 )
Generate plots and csvs for accuracy
Parameters
- ax (object): The matplotlib axis to plot on.
- save_folder (str): The folder to save the plots and csvs in.
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
generate_time_plots(self, ax, save_folder, extra_string):
2701 def generate_time_plots(self, ax, save_folder, extra_string): 2702 """ 2703 Generate plots and csvs for timing 2704 2705 Parameters 2706 ---------- 2707 ax : object 2708 The matplotlib axis to plot on. 2709 save_folder : str 2710 The folder to save the plots and csvs in. 2711 extra_string : str 2712 An extra string to append to the filenames. 2713 2714 Returns 2715 ------- 2716 None 2717 2718 """ 2719 if self.member_vars["manual_train_switch"]: 2720 ax.plot( 2721 np.arange(len(self.member_vars["n_train_times"])), 2722 self.member_vars["n_train_times"], 2723 label="Normal Epoch Train Times", 2724 color=_PAI_PALETTE["normal"], 2725 ) 2726 ax.plot( 2727 np.arange(len(self.member_vars["p_train_times"])), 2728 self.member_vars["p_train_times"], 2729 label="PAI Epoch Train Times", 2730 color=_PAI_PALETTE["dendrite"], 2731 ) 2732 ax.plot( 2733 np.arange(len(self.member_vars["n_val_times"])), 2734 self.member_vars["n_val_times"], 2735 label="Normal Epoch Val Times", 2736 color=_PAI_PALETTE["normal"], 2737 alpha=0.45, 2738 ) 2739 ax.plot( 2740 np.arange(len(self.member_vars["p_val_times"])), 2741 self.member_vars["p_val_times"], 2742 label="PAI Epoch Val Times", 2743 color=_PAI_PALETTE["dendrite"], 2744 alpha=0.45, 2745 ) 2746 2747 plt.title( 2748 save_folder + "/" + self.save_name + "times (by train() and eval())" 2749 ) 2750 plt.xlabel("Iteration") 2751 plt.ylabel("Epoch Time in Seconds ") 2752 ax.set_ylim(ymin=0) 2753 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2754 _pai_grid(ax) 2755 2756 pd1 = pd.DataFrame( 2757 { 2758 "Epochs": np.arange(len(self.member_vars["n_train_times"])), 2759 "Normal Epoch Train Times": self.member_vars["n_train_times"], 2760 } 2761 ) 2762 pd2 = pd.DataFrame( 2763 { 2764 "Epochs": np.arange(len(self.member_vars["p_train_times"])), 2765 "PAI Epoch Train Times": self.member_vars["p_train_times"], 2766 } 2767 ) 2768 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2769 2770 pd2 = pd.DataFrame( 2771 { 2772 "Epochs": np.arange(len(self.member_vars["n_val_times"])), 2773 "Normal Epoch Val Times": self.member_vars["n_val_times"], 2774 } 2775 ) 2776 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2777 2778 pd2 = pd.DataFrame( 2779 { 2780 "Epochs": np.arange(len(self.member_vars["p_val_times"])), 2781 "PAI Epoch Val Times": self.member_vars["p_val_times"], 2782 } 2783 ) 2784 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2785 2786 pd1.to_csv( 2787 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2788 index=False, 2789 ) 2790 del pd1, pd2 2791 else: 2792 ax.plot( 2793 np.arange(len(self.member_vars["n_epoch_times"])), 2794 self.member_vars["n_epoch_times"], 2795 label="Normal Epoch Times", 2796 color=_PAI_PALETTE["normal"], 2797 ) 2798 ax.plot( 2799 np.arange(len(self.member_vars["p_epoch_times"])), 2800 self.member_vars["p_epoch_times"], 2801 label="PAI Epoch Times", 2802 color=_PAI_PALETTE["dendrite"], 2803 ) 2804 2805 plt.title( 2806 save_folder + "/" + self.save_name + "times (by train() and eval())" 2807 ) 2808 plt.xlabel("Iteration") 2809 plt.ylabel("Epoch Time in Seconds ") 2810 ax.set_ylim(ymin=0) 2811 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2812 _pai_grid(ax) 2813 2814 pd1 = pd.DataFrame( 2815 { 2816 "Epochs": np.arange(len(self.member_vars["n_epoch_times"])), 2817 "Normal Epoch Times": self.member_vars["n_epoch_times"], 2818 } 2819 ) 2820 pd2 = pd.DataFrame( 2821 { 2822 "Epochs": np.arange(len(self.member_vars["p_epoch_times"])), 2823 "PAI Epoch Times": self.member_vars["p_epoch_times"], 2824 } 2825 ) 2826 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2827 2828 pd1.to_csv( 2829 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2830 index=False, 2831 ) 2832 del pd1, pd2 2833 2834 if self.values_per_train_epoch != -1 and self.values_per_val_epoch != -1: 2835 ax2 = ax.twinx() # Second axes sharing same x-axis 2836 ax2.set_ylabel("Single Datapoint Time in Seconds") 2837 2838 ax2.plot( 2839 np.arange(len(self.member_vars["n_train_times"])), 2840 np.array(self.member_vars["n_train_times"]) 2841 / self.values_per_train_epoch, 2842 linestyle="dashed", 2843 label="Normal Train Item Times", 2844 color=_PAI_PALETTE["normal"], 2845 ) 2846 ax2.plot( 2847 np.arange(len(self.member_vars["p_train_times"])), 2848 np.array(self.member_vars["p_train_times"]) 2849 / self.values_per_train_epoch, 2850 linestyle="dashed", 2851 label="PAI Train Item Times", 2852 color=_PAI_PALETTE["dendrite"], 2853 ) 2854 ax2.plot( 2855 np.arange(len(self.member_vars["n_val_times"])), 2856 np.array(self.member_vars["n_val_times"]) / self.values_per_val_epoch, 2857 linestyle="dashed", 2858 label="Normal Val Item Times", 2859 color=_PAI_PALETTE["normal"], 2860 alpha=0.45, 2861 ) 2862 ax2.plot( 2863 np.arange(len(self.member_vars["p_val_times"])), 2864 np.array(self.member_vars["p_val_times"]) / self.values_per_val_epoch, 2865 linestyle="dashed", 2866 label="PAI Val Item Times", 2867 color=_PAI_PALETTE["dendrite"], 2868 alpha=0.45, 2869 ) 2870 ax2.tick_params(axis="y") 2871 ax2.set_ylim(ymin=0) 2872 ax2.legend(bbox_to_anchor=(1.05, 1), loc="upper left")
Generate plots and csvs for timing
Parameters
- ax (object): The matplotlib axis to plot on.
- save_folder (str): The folder to save the plots and csvs in.
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
generate_learning_rate_plots(self, ax, save_folder, extra_string):
2874 def generate_learning_rate_plots(self, ax, save_folder, extra_string): 2875 """ 2876 Generate plots and csvs for learning rate 2877 2878 Parameters 2879 ---------- 2880 ax : object 2881 The matplotlib axis to plot on. 2882 save_folder : str 2883 The folder to save the plots and csvs in. 2884 extra_string : str 2885 An extra string to append to the filenames. 2886 2887 Returns 2888 ------- 2889 None 2890 2891 """ 2892 ax.plot( 2893 np.arange(len(self.member_vars["training_learning_rates"])), 2894 self.member_vars["training_learning_rates"], 2895 label="learning_rate", 2896 color=_PAI_PALETTE["learning_rate"], 2897 ) 2898 plt.title(save_folder + "/" + self.save_name + "learning_rate") 2899 plt.xlabel("Epochs") 2900 plt.ylabel("learning_rate") 2901 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2902 _pai_grid(ax) 2903 2904 pd1 = pd.DataFrame( 2905 { 2906 "Epochs": np.arange(len(self.member_vars["training_learning_rates"])), 2907 "learning_rate": self.member_vars["training_learning_rates"], 2908 } 2909 ) 2910 pd1.to_csv( 2911 save_folder + "/" + self.save_name + extra_string + "learning_rate.csv", 2912 index=False, 2913 ) 2914 del pd1
Generate plots and csvs for learning rate
Parameters
- ax (object): The matplotlib axis to plot on.
- save_folder (str): The folder to save the plots and csvs in.
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
get_current_pb_scores(self):
2916 def get_current_pb_scores(self): 2917 """ 2918 Get the latest best PBScore of each dendrite layer, the same numbers 2919 written to the Best PBScores csv. 2920 2921 Returns 2922 ------- 2923 dict 2924 Layer name to score. Empty outside of dendrite scoring phases, 2925 when no candidate dendrites are being scored. 2926 2927 """ 2928 if not self.member_vars["doing_pai"]: 2929 return {} 2930 if not GPA.pc.get_perforated_backpropagation(): 2931 return {} 2932 # Scores only advance while candidate dendrites are being trained 2933 if ( 2934 self.member_vars["mode"] != "p" 2935 and not GPA.pc.get_learn_dendrites_live() 2936 ): 2937 return {} 2938 2939 scores = {} 2940 for layer_id in range(len(self.neuron_module_vector)): 2941 if layer_id >= len(self.member_vars["best_scores"]): 2942 continue 2943 layer_scores = self.member_vars["best_scores"][layer_id] 2944 if len(layer_scores) == 0: 2945 continue 2946 score = layer_scores[-1] 2947 if hasattr(score, "item"): 2948 score = score.item() 2949 score = float(score) 2950 if math.isnan(score) or math.isinf(score): 2951 continue 2952 scores[self.neuron_module_vector[layer_id].name] = score 2953 return scores
Get the latest best PBScore of each dendrite layer, the same numbers written to the Best PBScores csv.
Returns
- dict: Layer name to score. Empty outside of dendrite scoring phases, when no candidate dendrites are being scored.
def
get_current_pb_scores_current(self):
2955 def get_current_pb_scores_current(self): 2956 """ 2957 Like :meth:`get_current_pb_scores` but returns the *current* (not best) 2958 PBScore of each dendrite layer - ``current_scores[layer][-1]`` - the 2959 dashed series in ``generate_dendrite_learning_plots``. 2960 2961 Returns 2962 ------- 2963 dict 2964 Layer name to score. Empty outside of dendrite scoring phases. 2965 """ 2966 if not self.member_vars["doing_pai"]: 2967 return {} 2968 if not GPA.pc.get_perforated_backpropagation(): 2969 return {} 2970 if ( 2971 self.member_vars["mode"] != "p" 2972 and not GPA.pc.get_learn_dendrites_live() 2973 ): 2974 return {} 2975 2976 scores = {} 2977 for layer_id in range(len(self.neuron_module_vector)): 2978 if layer_id >= len(self.member_vars["current_scores"]): 2979 continue 2980 layer_scores = self.member_vars["current_scores"][layer_id] 2981 if len(layer_scores) == 0: 2982 continue 2983 score = layer_scores[-1] 2984 if hasattr(score, "item"): 2985 score = score.item() 2986 score = float(score) 2987 if math.isnan(score) or math.isinf(score): 2988 continue 2989 scores[self.neuron_module_vector[layer_id].name] = score 2990 return scores
Like get_current_pb_scores() but returns the current (not best)
PBScore of each dendrite layer - current_scores[layer][-1] - the
dashed series in generate_dendrite_learning_plots.
Returns
- dict: Layer name to score. Empty outside of dendrite scoring phases.
def
generate_dendrite_learning_plots(self, ax, save_folder, extra_string):
2992 def generate_dendrite_learning_plots(self, ax, save_folder, extra_string): 2993 """ 2994 Generate dendrite score plots for the tracker. 2995 Also saves csv files associated with the plots. 2996 2997 Parameters 2998 ---------- 2999 ax : matplotlib.axes.Axes 3000 Axis used for plotting dendrite-learning curves. 3001 save_folder : str 3002 Directory where plot images and CSV summaries are written. 3003 extra_string : str 3004 Filename suffix used to distinguish this output set. 3005 3006 Returns 3007 ------- 3008 None 3009 Saves plots and score CSV files to disk. 3010 """ 3011 if self.member_vars["doing_pai"]: 3012 pd1 = None 3013 pd2 = None 3014 num_colors = len(self.neuron_module_vector) 3015 3016 layer_colors = [ 3017 _PAI_DENDRITE_CYCLE[i % len(_PAI_DENDRITE_CYCLE)] 3018 for i in range(num_colors) 3019 ] 3020 3021 for layer_id in range(len(self.neuron_module_vector)): 3022 color = layer_colors[layer_id] 3023 ax.plot( 3024 np.arange(len(self.member_vars["best_scores"][layer_id])), 3025 self.member_vars["best_scores"][layer_id], 3026 label=self.neuron_module_vector[layer_id].name, 3027 color=color, 3028 ) 3029 3030 pd2 = pd.DataFrame( 3031 { 3032 "Epochs": np.arange( 3033 len(self.member_vars["best_scores"][layer_id]) 3034 ), 3035 f"Best ever for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 3036 "best_scores" 3037 ][ 3038 layer_id 3039 ], 3040 } 3041 ) 3042 3043 if pd1 is None: 3044 pd1 = pd2 3045 else: 3046 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 3047 3048 if len(self.member_vars["current_scores"][layer_id]) != 0: 3049 ax.plot( 3050 np.arange(len(self.member_vars["current_scores"][layer_id])), 3051 self.member_vars["current_scores"][layer_id], 3052 label=f"Current:{self.neuron_module_vector[layer_id].name}", 3053 color=color, 3054 linestyle="--", 3055 ) 3056 3057 pd2 = pd.DataFrame( 3058 { 3059 "Epochs": np.arange( 3060 len(self.member_vars["current_scores"][layer_id]) 3061 ), 3062 f"Best current for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 3063 "current_scores" 3064 ][ 3065 layer_id 3066 ], 3067 } 3068 ) 3069 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 3070 3071 plt.title(save_folder + "/" + self.save_name + " Best PBScores") 3072 plt.xlabel("Epochs") 3073 plt.ylabel("Best PBScore") 3074 ax.legend( 3075 bbox_to_anchor=(1.05, 1), 3076 loc="upper left", 3077 ncol=max(1, math.ceil(len(self.neuron_module_vector) / 30)), 3078 ) 3079 _pai_grid(ax) 3080 for switcher in self.member_vars["p_switch_epochs"]: 3081 plt.axvline( 3082 x=switcher, ymin=0, ymax=1, color=_PAI_PALETTE["dendrite"] 3083 ) 3084 3085 if self.member_vars["mode"] == "p": 3086 missed_time = ( 3087 self.member_vars["num_epochs_run"] 3088 - self.member_vars["epoch_last_improved"] 3089 ) 3090 plt.axvline( 3091 x=(len(self.member_vars["best_scores"][0]) - (missed_time + 1)), 3092 ymin=0, 3093 ymax=1, 3094 color=_PAI_PALETTE["normal"], 3095 ) 3096 3097 # pd1 here will be none if no PB layers are created 3098 if pd1 is not None: 3099 pd1.to_csv( 3100 save_folder 3101 + "/" 3102 + self.save_name 3103 + extra_string 3104 + "Best PBScores.csv", 3105 index=False, 3106 ) 3107 del pd1, pd2
Generate dendrite score plots for the tracker. Also saves csv files associated with the plots.
Parameters
- ax (matplotlib.axes.Axes): Axis used for plotting dendrite-learning curves.
- save_folder (str): Directory where plot images and CSV summaries are written.
- extra_string (str): Filename suffix used to distinguish this output set.
Returns
- None: Saves plots and score CSV files to disk.
def
generate_extra_csv_files(self, save_folder, extra_string):
3109 def generate_extra_csv_files(self, save_folder, extra_string): 3110 """ 3111 Generate additional csvs 3112 3113 Parameters 3114 ---------- 3115 save_folder : str 3116 The folder to save the plots and csvs in. 3117 extra_string : str 3118 An extra string to append to the filenames. 3119 3120 Returns 3121 ------- 3122 None 3123 3124 """ 3125 pd1 = pd.DataFrame( 3126 { 3127 "Switch Number": np.arange(len(self.member_vars["switch_epochs"])), 3128 "Switch Epoch": self.member_vars["switch_epochs"], 3129 } 3130 ) 3131 pd1.to_csv( 3132 save_folder + "/" + self.save_name + extra_string + "switch_epochs.csv", 3133 index=False, 3134 ) 3135 del pd1 3136 3137 pd1 = pd.DataFrame( 3138 { 3139 "Switch Number": np.arange(len(self.member_vars["param_counts"])), 3140 "Param Count": self.member_vars["param_counts"], 3141 } 3142 ) 3143 pd1.to_csv( 3144 save_folder + "/" + self.save_name + extra_string + "param_counts.csv", 3145 index=False, 3146 ) 3147 del pd1 3148 3149 """ 3150 Create best_arch_scores.csv file 3151 When working with dendrites there is a tradeoff between additional param count and score improvement. 3152 This file will help track that tradeoff by recording the best scores for all extra_scores 3153 and extra_scores_without_graphing for each architecture version. 3154 The scores recorded here are from the epoch when the best validation score was found 3155 within each switch_epoch boundary. 3156 """ 3157 switch_counts = len(self.member_vars["switch_epochs"]) 3158 best_valid = [] 3159 associated_params = [] 3160 3161 # Initialize dictionaries to store best scores for each extra score type 3162 best_extra_scores = {} 3163 for score_name in self.member_vars["extra_scores"]: 3164 best_extra_scores[score_name] = [] 3165 for score_name in self.member_vars["extra_scores_without_graphing"]: 3166 best_extra_scores[score_name] = [] 3167 3168 for switch in range(0, switch_counts, 2): 3169 start_index = 0 3170 if switch != 0: 3171 start_index = self.member_vars["switch_epochs"][switch - 1] + 1 3172 end_index = self.member_vars["switch_epochs"][switch] + 1 3173 3174 if GPA.pai_tracker.member_vars["maximizing_score"]: 3175 best_valid_index = start_index + np.argmax( 3176 self.member_vars["accuracies"][start_index:end_index] 3177 ) 3178 else: 3179 best_valid_index = start_index + np.argmin( 3180 self.member_vars["accuracies"][start_index:end_index] 3181 ) 3182 3183 best_valid_score = self.member_vars["accuracies"][best_valid_index] 3184 best_valid.append(best_valid_score) 3185 3186 # Get corresponding scores from all extra_scores 3187 for score_name in self.member_vars["extra_scores"]: 3188 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 3189 best_extra_scores[score_name].append( 3190 self.member_vars["extra_scores"][score_name][best_valid_index] 3191 ) 3192 else: 3193 best_extra_scores[score_name].append(None) 3194 3195 # Get corresponding scores from all extra_scores_without_graphing 3196 for score_name in self.member_vars["extra_scores_without_graphing"]: 3197 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 3198 best_extra_scores[score_name].append( 3199 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 3200 ) 3201 else: 3202 best_extra_scores[score_name].append(None) 3203 3204 if self.member_vars["doing_pai"]: 3205 associated_params.append(self.member_vars["param_counts"][switch]) 3206 else: 3207 associated_params.append(self.member_vars["param_counts"][-1]) 3208 3209 # If in neuron training mode but not the very first epoch 3210 if self.member_vars["mode"] == "n" and ( 3211 (len(self.member_vars["switch_epochs"]) == 0) 3212 or ( 3213 self.member_vars["switch_epochs"][-1] + 1 3214 != len(self.member_vars["accuracies"]) 3215 ) 3216 ): 3217 start_index = 0 3218 if len(self.member_vars["switch_epochs"]) != 0: 3219 start_index = self.member_vars["switch_epochs"][-1] + 1 3220 3221 if GPA.pai_tracker.member_vars["maximizing_score"]: 3222 best_valid_index = start_index + np.argmax( 3223 self.member_vars["accuracies"][start_index:] 3224 ) 3225 else: 3226 best_valid_index = start_index + np.argmin( 3227 self.member_vars["accuracies"][start_index:] 3228 ) 3229 3230 best_valid_score = self.member_vars["accuracies"][best_valid_index] 3231 best_valid.append(best_valid_score) 3232 3233 # Get corresponding scores from all extra_scores 3234 for score_name in self.member_vars["extra_scores"]: 3235 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 3236 best_extra_scores[score_name].append( 3237 self.member_vars["extra_scores"][score_name][best_valid_index] 3238 ) 3239 else: 3240 best_extra_scores[score_name].append(None) 3241 3242 # Get corresponding scores from all extra_scores_without_graphing 3243 for score_name in self.member_vars["extra_scores_without_graphing"]: 3244 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 3245 best_extra_scores[score_name].append( 3246 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 3247 ) 3248 else: 3249 best_extra_scores[score_name].append(None) 3250 3251 associated_params.append(self.member_vars["param_counts"][-1]) 3252 3253 # Build dataframe with all columns 3254 csv_data = { 3255 "Param Counts": associated_params, 3256 "Max Valid Scores": best_valid, 3257 } 3258 3259 # Add columns for each extra score 3260 for score_name in best_extra_scores: 3261 csv_data[score_name] = best_extra_scores[score_name] 3262 3263 pd1 = pd.DataFrame(csv_data) 3264 pd1.to_csv( 3265 save_folder + "/" + self.save_name + extra_string + "_best_arch_scores.csv", 3266 index=False, 3267 ) 3268 del pd1
Generate additional csvs
Parameters
- save_folder (str): The folder to save the plots and csvs in.
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
save_graphs(self, extra_string=''):
3270 def save_graphs(self, extra_string=""): 3271 """ 3272 Save graphs and csvs for all the values the tracker records 3273 3274 Parameters 3275 ---------- 3276 extra_string : str 3277 An extra string to append to the filenames. 3278 3279 Returns 3280 ------- 3281 None 3282 3283 """ 3284 # If running DDP only save with rank 0 3285 if "RANK" in os.environ: 3286 if int(os.environ["RANK"]) != 0: 3287 return 3288 if not self.making_graphs: 3289 return 3290 3291 save_folder = "./" + self.save_name + "/" 3292 3293 plt.ioff() 3294 fig = plt.figure(figsize=(28, 14)) 3295 3296 # Plot with accuracy scores 3297 ax = plt.subplot(221) 3298 self.generate_accuracy_plots(ax, save_folder, extra_string) 3299 3300 # Plot dendrite learning scores 3301 ax = plt.subplot(222) 3302 self.generate_dendrite_learning_plots(ax, save_folder, extra_string) 3303 3304 if GPA.pc.get_drawing_extra_graphs(): 3305 # Plot learning rates for each training epoch 3306 ax = plt.subplot(223) 3307 self.generate_learning_rate_plots(ax, save_folder, extra_string) 3308 3309 # Plot the times for each training epoch 3310 ax = plt.subplot(224) 3311 self.generate_time_plots(ax, save_folder, extra_string) 3312 3313 # Generate extra CSV files 3314 self.generate_extra_csv_files(save_folder, extra_string) 3315 3316 fig.tight_layout() 3317 plt.savefig(save_folder + "/" + self.save_name + extra_string + ".png") 3318 plt.close("all")
Save graphs and csvs for all the values the tracker records
Parameters
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
add_loss(self, loss):
3320 def add_loss(self, loss): 3321 """Add loss to tracking vectors. 3322 3323 Parameters 3324 ---------- 3325 loss : float or int 3326 The loss value to add. 3327 3328 Returns 3329 ------- 3330 None 3331 3332 """ 3333 if not isinstance(loss, (float, int)): 3334 loss = loss.item() 3335 self.member_vars["training_loss"].append(loss)
Add loss to tracking vectors.
Parameters
- loss (float or int): The loss value to add.
Returns
- None
def
add_learning_rate(self, learning_rate):
3337 def add_learning_rate(self, learning_rate): 3338 """Add learning rate to tracking vectors. 3339 3340 Parameters 3341 ---------- 3342 learning_rate : float or int 3343 The learning rate value to add. 3344 3345 Returns 3346 ------- 3347 None 3348 3349 """ 3350 if not isinstance(learning_rate, (float, int)): 3351 learning_rate = learning_rate.item() 3352 self.member_vars["training_learning_rates"].append(learning_rate)
Add learning rate to tracking vectors.
Parameters
- learning_rate (float or int): The learning rate value to add.
Returns
- None
def
add_extra_score(self, score, extra_score_name):
3354 def add_extra_score(self, score, extra_score_name): 3355 """Add extra score to tracking vectors. 3356 3357 Parameters 3358 ---------- 3359 score : float or int 3360 The score value to add. 3361 3362 extra_score_name : str 3363 The name of the extra score. 3364 3365 Returns 3366 ------- 3367 None 3368 3369 """ 3370 if not isinstance(score, (float, int)): 3371 try: 3372 score = score.item() 3373 except: 3374 print( 3375 "Scores added for Perforated Backpropagation should be " 3376 "float, int, or tensor, yours is a:" 3377 ) 3378 print(type(score)) 3379 pdb.set_trace() 3380 3381 if GPA.pc.get_verbose(): 3382 print(f"Adding extra score {extra_score_name} of {float(score)}") 3383 3384 if extra_score_name not in self.member_vars["extra_scores"]: 3385 self.member_vars["extra_scores"][extra_score_name] = [] 3386 self.member_vars["extra_scores"][extra_score_name].append(score) 3387 3388 if self.member_vars["mode"] == "n": 3389 if extra_score_name not in self.member_vars["n_extra_scores"]: 3390 self.member_vars["n_extra_scores"][extra_score_name] = [] 3391 self.member_vars["n_extra_scores"][extra_score_name].append(score)
Add extra score to tracking vectors.
Parameters
- score (float or int): The score value to add.
- extra_score_name (str): The name of the extra score.
Returns
- None
def
add_extra_score_without_graphing(self, score, extra_score_name):
3393 def add_extra_score_without_graphing(self, score, extra_score_name): 3394 """Add extra score without graphing to tracking vectors. 3395 3396 Parameters 3397 ---------- 3398 score : float or int 3399 The score value to add. 3400 3401 extra_score_name : str 3402 The name of the extra score. 3403 3404 Returns 3405 ------- 3406 None 3407 3408 """ 3409 if not isinstance(score, (float, int)): 3410 try: 3411 score = score.item() 3412 except: 3413 print( 3414 "Scores added for Perforated Backpropagation should be " 3415 "float, int, or tensor, yours is a:" 3416 ) 3417 print(type(score)) 3418 print("in add_extra_score_without_graphing") 3419 pdb.set_trace() 3420 3421 if GPA.pc.get_verbose(): 3422 print(f"Adding extra score {extra_score_name} of {float(score)}") 3423 3424 if extra_score_name not in self.member_vars["extra_scores_without_graphing"]: 3425 self.member_vars["extra_scores_without_graphing"][extra_score_name] = [] 3426 self.member_vars["extra_scores_without_graphing"][extra_score_name].append( 3427 score 3428 )
Add extra score without graphing to tracking vectors.
Parameters
- score (float or int): The score value to add.
- extra_score_name (str): The name of the extra score.
Returns
- None
def
add_test_score(self, score, extra_score_name):
3430 def add_test_score(self, score, extra_score_name): 3431 """Add test score to tracking vectors. 3432 3433 Parameters 3434 ---------- 3435 score : float or int 3436 The score value to add. 3437 3438 extra_score_name : str 3439 The name of the extra score. 3440 3441 Returns 3442 ------- 3443 None 3444 3445 Notes 3446 ----- 3447 This function is a wrapper around `add_extra_score` that separates 3448 test score for adding to best_arch_scores.csv. 3449 3450 """ 3451 self.add_extra_score(score, extra_score_name) 3452 3453 if not isinstance(score, (float, int)): 3454 try: 3455 score = score.item() 3456 except: 3457 print( 3458 "Scores added for Perforated Backpropagation should be " 3459 "float, int, or tensor, yours is a:" 3460 ) 3461 print(type(score)) 3462 print("in add_test_score") 3463 pdb.set_trace() 3464 3465 if GPA.pc.get_verbose(): 3466 print(f"Adding test score {extra_score_name} of {float(score)}") 3467 self.member_vars["test_scores"].append(score)
Add test score to tracking vectors.
Parameters
- score (float or int): The score value to add.
- extra_score_name (str): The name of the extra score.
Returns
- None
Notes
This function is a wrapper around add_extra_score that separates
test score for adding to best_arch_scores.csv.
def
add_validation_score(self, accuracy, net, force_switch=False):
3469 def add_validation_score(self, accuracy, net, force_switch=False): 3470 """Function to add the validation score. 3471 3472 This is complex because it determines neuron and dendrite switching. 3473 3474 Parameters 3475 ---------- 3476 accuracy : float or int 3477 The accuracy or loss value to add. 3478 net : object 3479 The neural network model. 3480 force_switch : bool, optional 3481 Whether to force a switch, by default False. 3482 3483 Returns 3484 ------- 3485 net : object 3486 The potentially modified neural network model. 3487 training_complete : bool 3488 Whether training is complete. 3489 restructured : bool 3490 Whether the model has been restructured. 3491 3492 Notes 3493 ----- 3494 WARNING: Do not call self anywhere in this function. When systems 3495 get loaded the actual tracker you are working with can change. 3496 """ 3497 3498 _pai_log("info", f"Adding validation score {accuracy:.8f}") 3499 3500 update_learning_rate() 3501 update_param_count(net) 3502 3503 accuracy = check_input_problems(net, accuracy) 3504 3505 if len(GPA.pai_tracker.member_vars["switch_epochs"]) == 0: 3506 epochs_since_cycle_switch = GPA.pai_tracker.member_vars["num_epochs_run"] 3507 else: 3508 epochs_since_cycle_switch = ( 3509 GPA.pai_tracker.member_vars["num_epochs_run"] 3510 - GPA.pai_tracker.member_vars["switch_epochs"][-1] 3511 ) 3512 3513 update_running_accuracy(accuracy, epochs_since_cycle_switch) 3514 if GPA.pc.get_perforated_backpropagation(): 3515 TPB.update_pb_scores(self) 3516 3517 # Captured before any switch below flips the mode and reloads scores. 3518 # epoch_phase is sampled here too so the emitted epoch event's phase and 3519 # its pb_scores always describe the same side of a switch boundary. 3520 epoch_phase = GPA.pai_tracker.member_vars["mode"] 3521 epoch_pb_scores = self.get_current_pb_scores() 3522 epoch_pb_scores_current = self.get_current_pb_scores_current() 3523 3524 GPA.pai_tracker.stop_epoch(internal_call=True) 3525 3526 # If it is neuron training mode 3527 if ( 3528 GPA.pai_tracker.member_vars["mode"] == "n" 3529 or GPA.pc.get_learn_dendrites_live() 3530 ): 3531 check_new_best(net, accuracy, epochs_since_cycle_switch) 3532 elif GPA.pc.get_perforated_backpropagation(): 3533 TPB.check_best_pai_score_improvement() 3534 3535 # Save the latest model 3536 if GPA.pc.get_test_saves(): 3537 UPA.save_system(net, GPA.pc.get_save_name(), "latest") 3538 if GPA.pc.get_pai_saves(): 3539 UPA.pai_save_system(net, GPA.pc.get_save_name(), "latest") 3540 3541 restructuring_status_value = NO_MODEL_UPDATE 3542 # If it is time to switch based on scores and counter or a manual switch 3543 if GPA.pai_tracker.switch_time() or force_switch: 3544 # If testing dendrite capacity switch after enough dendrites added 3545 if ( 3546 (GPA.pai_tracker.member_vars["mode"] == "n") 3547 and (GPA.pai_tracker.member_vars["num_dendrites_added"] > 2) 3548 and GPA.pc.get_testing_dendrite_capacity() 3549 ): 3550 GPA.pai_tracker.save_graphs() 3551 _pai_log( 3552 "info", 3553 "Successfully added 3 dendrites with GPA.pc.set_testing_dendrite_capacity(True) (default). " 3554 "You may now set that to False and run a real experiment.", 3555 ) 3556 return net, False, True 3557 3558 # If doing neuron training but this dendrite count didn't improve 3559 if ( 3560 (GPA.pai_tracker.member_vars["mode"] == "n") 3561 or GPA.pc.get_learn_dendrites_live() 3562 ) and (GPA.pai_tracker.member_vars["current_n_set_global_best"] is False): 3563 new_restructuring_status_value, net = process_no_improvement(net) 3564 # if this was the final try return that training is complete 3565 if new_restructuring_status_value == TRAINING_COMPLETE: 3566 if _dashboard_emitter is not None: 3567 _dashboard_emitter.emit_run_end( 3568 GPA.pc, 3569 epoch_last_improved=GPA.pai_tracker.member_vars[ 3570 "epoch_last_improved" 3571 ], 3572 global_best_score=GPA.pai_tracker.member_vars[ 3573 "global_best_validation_score" 3574 ], 3575 ) 3576 return net, True, True 3577 else: 3578 restructuring_status_value = update_restructuring_status( 3579 restructuring_status_value, new_restructuring_status_value 3580 ) 3581 # Else if did improve, do a normal switch process 3582 else: 3583 if GPA.pc.get_verbose(): 3584 print( 3585 f"Calling switch_mode with " 3586 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]}, ' 3587 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 3588 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]}, ' 3589 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_value"]},' 3590 f'{GPA.pc.get_max_dendrites()},' 3591 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]},' 3592 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"]},' 3593 ) 3594 import pdb; pdb.set_trace 3595 # If the max number of dendrites has been hit or not doing pai and adding dendtites 3596 # then return rather than adding more 3597 if ( 3598 (GPA.pai_tracker.member_vars["mode"] == "n") 3599 and ( 3600 GPA.pc.get_max_dendrites() 3601 == GPA.pai_tracker.member_vars["num_dendrites_added"] 3602 ) 3603 ) or (GPA.pai_tracker.member_vars["doing_pai"] is False): 3604 if GPA.pc.get_verbose(): 3605 print( 3606 "Max dendrites reached or not doing PAI, finishing training" 3607 ) 3608 net = process_final_network(net) 3609 # Increment integrated if we have dendrites (means they're integrated) 3610 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3611 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3612 _pai_log("info", f"Final dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3613 if _dashboard_emitter is not None: 3614 _dashboard_emitter.emit_dendrite_added( 3615 GPA.pc, 3616 epoch_index=len( 3617 GPA.pai_tracker.member_vars["accuracies"] 3618 ) - 1, 3619 num_dendrites_integrated=GPA.pai_tracker.member_vars[ 3620 "num_dendrites_integrated" 3621 ], 3622 param_count=UPA.count_params(net), 3623 ) 3624 if _dashboard_emitter is not None: 3625 _dashboard_emitter.emit_run_end( 3626 GPA.pc, 3627 epoch_last_improved=GPA.pai_tracker.member_vars[ 3628 "epoch_last_improved" 3629 ], 3630 global_best_score=GPA.pai_tracker.member_vars[ 3631 "global_best_validation_score" 3632 ], 3633 ) 3634 return net, True, True 3635 3636 # Otherwise if its neuron training mode reset the counter of failed dendrites 3637 # Check if we should increment integrated count BEFORE change_learning_modes loads old state 3638 should_increment_integrated = False 3639 if GPA.pai_tracker.member_vars["mode"] == "n": 3640 GPA.pai_tracker.member_vars["num_dendrite_tries"] = 0 3641 if GPA.pc.get_verbose(): 3642 print( 3643 "Adding new dendrites without resetting which means " 3644 "the last ones improved. Resetting num_dendrite_tries" 3645 ) 3646 # Remember to increment after change_learning_modes (which loads old tracker state) 3647 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3648 should_increment_integrated = True 3649 3650 GPA.pai_tracker.save_graphs( 3651 f'_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}' 3652 ) 3653 3654 if GPA.pc.get_test_saves(): 3655 UPA.save_system( 3656 net, 3657 GPA.pc.get_save_name(), 3658 f'beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3659 ) 3660 # Copy current best model from this set of dendrites 3661 # If running DDP only copy with rank 0 3662 if "RANK" not in os.environ or int(os.environ["RANK"]) == 0: 3663 shutil.copyfile( 3664 f"{GPA.pc.get_save_name()}/best_model.pt", 3665 f'{GPA.pc.get_save_name()}/best_model_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}.pt', 3666 ) 3667 3668 if GPA.pc.get_dashboard_debug(): 3669 _pai_log( 3670 "debug", 3671 "[emitter/finding4] before change_learning_modes: " 3672 f'len(accuracies)={len(GPA.pai_tracker.member_vars["accuracies"])} ' 3673 f'num_epochs_run={GPA.pai_tracker.member_vars["num_epochs_run"]}', 3674 ) 3675 net = UPA.change_learning_modes( 3676 net, 3677 GPA.pc.get_save_name(), 3678 "best_model", 3679 GPA.pai_tracker.member_vars["doing_pai"], 3680 ) 3681 restructuring_status_value = NETWORK_RESTRUCTURED 3682 if GPA.pc.get_dashboard_debug(): 3683 _pai_log( 3684 "debug", 3685 "[emitter/finding4] after change_learning_modes: " 3686 f'len(accuracies)={len(GPA.pai_tracker.member_vars["accuracies"])} ' 3687 f'num_epochs_run={GPA.pai_tracker.member_vars["num_epochs_run"]}', 3688 ) 3689 3690 # Now increment after change_learning_modes has loaded the best model 3691 # This ensures the increment persists and doesn't get overwritten 3692 if should_increment_integrated: 3693 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3694 _pai_log("info", f"Dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3695 if _dashboard_emitter is not None: 3696 _dashboard_emitter.emit_dendrite_added( 3697 GPA.pc, 3698 epoch_index=len( 3699 GPA.pai_tracker.member_vars["accuracies"] 3700 ) - 1, 3701 num_dendrites_integrated=GPA.pai_tracker.member_vars[ 3702 "num_dendrites_integrated" 3703 ], 3704 param_count=UPA.count_params(net), 3705 ) 3706 3707 # If restructured is true, clear scheduler/optimizer before saving 3708 if restructuring_status_value != NETWORK_RESTRUCTURED: 3709 print( 3710 "Restructured should always be triggered here, let us know if you encounter this situation" 3711 ) 3712 pdb.set_trace() 3713 3714 # Since there is a restructuring optimizer and scheduler must be reinitialized after return 3715 GPA.pai_tracker.clear_optimizer_and_scheduler() 3716 3717 # Save the model from after the switch 3718 UPA.save_system( 3719 net, 3720 GPA.pc.get_save_name(), 3721 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3722 ) 3723 3724 # If not time to switch and you have a scheduler, perform the update step 3725 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 3726 new_restructuring_status_value, net = process_scheduler_update( 3727 net, accuracy, epochs_since_cycle_switch 3728 ) 3729 restructuring_status_value = update_restructuring_status( 3730 restructuring_status_value, new_restructuring_status_value 3731 ) 3732 3733 GPA.pai_tracker.start_epoch(internal_call=True) 3734 if _dashboard_emitter is not None: 3735 _mv = GPA.pai_tracker.member_vars 3736 _lr = _mv["training_learning_rates"][-1] if _mv["training_learning_rates"] else None 3737 # epoch_index is the committed coordinate: len(accuracies) - 1, 3738 # captured after update_running_accuracy appends and after any reload 3739 # truncation in switch handling. Independent of num_epochs_run. 3740 _epoch_index = len(_mv["accuracies"]) - 1 3741 if GPA.pc.get_dashboard_debug(): 3742 _pai_log( 3743 "debug", 3744 "[emitter/finding4] emit_epoch: " 3745 f"epoch_index={_epoch_index} " 3746 f'true_epoch={_mv["num_epochs_run"]} ' 3747 f'restructured={restructuring_status_value == NETWORK_RESTRUCTURED} ' 3748 f'tail_accuracies={_mv["accuracies"][-4:]}', 3749 ) 3750 # Phase captured at the top of the method (pre-switch), matching the 3751 # point pb_scores were sampled - see Finding 2, ticket 06. 3752 _phase = epoch_phase 3753 # scores: validation (reported), running (checkpointed member var), 3754 # then every add_extra_score name verbatim. 3755 _scores = { 3756 "validation": accuracy, 3757 "running": _mv["running_accuracies"][-1] if _mv["running_accuracies"] else None, 3758 } 3759 for _name, _vals in _mv["extra_scores"].items(): 3760 if _vals: 3761 _scores[_name] = _vals[-1] 3762 _n_times = _mv["n_epoch_times"] or [(_mv["n_train_times"][-1] + _mv["n_val_times"][-1]) if (_mv["n_train_times"] and _mv["n_val_times"]) else None] 3763 _p_times = _mv["p_epoch_times"] or [(_mv["p_train_times"][-1] + _mv["p_val_times"][-1]) if (_mv["p_train_times"] and _mv["p_val_times"]) else None] 3764 _dashboard_emitter.emit_epoch( 3765 GPA.pc, 3766 epoch_index=_epoch_index, 3767 phase=_phase, 3768 scores=_scores, 3769 learning_rate=_lr, 3770 normal_time=_n_times[-1] if _phase == "n" else None, 3771 pai_time=_p_times[-1] if _phase == "p" else None, 3772 pb_scores=epoch_pb_scores, 3773 pb_scores_current=epoch_pb_scores_current, 3774 ) 3775 GPA.pai_tracker.save_graphs() 3776 3777 if restructuring_status_value == NETWORK_RESTRUCTURED: 3778 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 3779 GPA.pai_tracker.member_vars["num_epochs_run"] 3780 ) 3781 if GPA.pc.get_verbose(): 3782 print( 3783 f"Setting epoch last improved to " 3784 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]}' 3785 ) 3786 3787 now = datetime.now() 3788 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 3789 3790 if GPA.pc.get_verbose(): 3791 print("Not saving restructure right now") 3792 3793 """ 3794 This block of code helped with a save issue with safetensors and huggingface, but it breaks DDP. 3795 Temporarily removing it to avoid DDP issues, but if you encounter save issues try adding it back in. 3796 for param in net.parameters(): 3797 param.data = param.data.contiguous() 3798 """ 3799 if GPA.pc.get_verbose(): 3800 print( 3801 f"Completed adding score. Restructured is {restructuring_status_value}, " 3802 f"\ncurrent switch list is:" 3803 ) 3804 print(GPA.pai_tracker.member_vars["switch_epochs"]) 3805 3806 if _dashboard_emitter is not None and restructuring_status_value == NETWORK_RESTRUCTURED: 3807 _param_count = UPA.count_params(net) 3808 _dashboard_emitter.emit_switch( 3809 GPA.pc, 3810 # switch_epochs was already appended in change_learning_modes, 3811 # so subtract 1 for a 0-based ordinal (matches matplotlib's 3812 # np.arange(len(switch_epochs)) labelling) - Finding 1, ticket 06. 3813 switch_ordinal=len(GPA.pai_tracker.member_vars["switch_epochs"]) - 1, 3814 epoch_index=len(GPA.pai_tracker.member_vars["accuracies"]) - 1, 3815 param_count=_param_count, 3816 switch_type=GPA.pai_tracker.member_vars["mode"], 3817 ) 3818 3819 # Always False for training complete if nothing triggered that training is over 3820 return net, restructuring_status_value, False
Function to add the validation score.
This is complex because it determines neuron and dendrite switching.
Parameters
- accuracy (float or int): The accuracy or loss value to add.
- net (object): The neural network model.
- force_switch (bool, optional): Whether to force a switch, by default False.
Returns
- net (object): The potentially modified neural network model.
- training_complete (bool): Whether training is complete.
- restructured (bool): Whether the model has been restructured.
Notes
WARNING: Do not call self anywhere in this function. When systems get loaded the actual tracker you are working with can change.
def
clear_all_processors(self):
3822 def clear_all_processors(self): 3823 """Clear all processors from modules. 3824 3825 Parameters 3826 ---------- 3827 None 3828 3829 Returns 3830 ------- 3831 None 3832 This function does not return a value. 3833 """ 3834 for module in self.neuron_module_vector: 3835 module.clear_processors()
Clear all processors from modules.
Parameters
- None
Returns
- None: This function does not return a value.
def
create_new_dendrite_module(self):
3837 def create_new_dendrite_module(self): 3838 """Add dendrite module to all neuron modules. 3839 3840 Parameters 3841 ---------- 3842 None 3843 3844 Returns 3845 ------- 3846 None 3847 This function does not return a value. 3848 """ 3849 for module in self.neuron_module_vector: 3850 module.create_new_dendrite_module()
Add dendrite module to all neuron modules.
Parameters
- None
Returns
- None: This function does not return a value.
def
set_create_dendrite_global(self, fn):
3852 def set_create_dendrite_global(self, fn): 3853 """Call set_create_dendrite(fn) on every tracked PAINeuronModule.""" 3854 for module in self.neuron_module_vector: 3855 module.set_create_dendrite(fn)
Call set_create_dendrite(fn) on every tracked PAINeuronModule.
def
set_dendrite_loss_fn_global(self, fn):
3857 def set_dendrite_loss_fn_global(self, fn): 3858 """Set the global dendrite loss function used by all dendrite modules.""" 3859 from perforatedbp import modules_pbp as MPB 3860 MPB.dendrite_loss_fn = fn
Set the global dendrite loss function used by all dendrite modules.
def
apply_pb_grads(self):
3862 def apply_pb_grads(self): 3863 """Apply perforated backpropagation gradients to all modules. 3864 3865 Parameters 3866 ---------- 3867 None 3868 3869 Returns 3870 ------- 3871 None 3872 This function does not return a value. 3873 """ 3874 if self.member_vars["mode"] == "p": 3875 for module in self.neuron_module_vector: 3876 module.apply_pb_grads()
Apply perforated backpropagation gradients to all modules.
Parameters
- None
Returns
- None: This function does not return a value.
def
apply_pb_zero(self):
3878 def apply_pb_zero(self): 3879 """Apply perforated backpropagation zero gradients to all modules. 3880 3881 Parameters 3882 ---------- 3883 None 3884 3885 Returns 3886 ------- 3887 None 3888 This function does not return a value. 3889 """ 3890 if self.member_vars["mode"] == "p": 3891 for module in self.neuron_module_vector: 3892 module.apply_pb_zero()
Apply perforated backpropagation zero gradients to all modules.
Parameters
- None
Returns
- None: This function does not return a value.