perforatedai.tracker_perforatedai
1# Copyright (c) 2025 Perforated AI 2 3import io 4import math 5import os 6import shutil 7import sys 8import time 9from datetime import datetime 10from pydoc import locate 11 12import matplotlib as mpl 13import matplotlib.pyplot as plt 14import numpy as np 15import pandas as pd 16import torch 17import torch.nn as nn 18import torch.nn.functional as F 19import torch.nn.init as init 20import pdb 21 22from perforatedai import globals_perforatedai as GPA 23from perforatedai import modules_perforatedai as PA 24from perforatedai import utils_perforatedai as UPA 25 26 27try: 28 from perforatedbp import tracker_pbp as TPB 29except ModuleNotFoundError: 30 pass # Module not found, pass silently 31except ImportError as e: 32 print(f"Import error occurred: {e}") 33 34mpl.use("Agg") 35 36# Status constants for restructuring during add_validation_score 37NO_MODEL_UPDATE = 0 38NETWORK_RESTRUCTURED = 1 39TRAINING_COMPLETE = 2 40 41# Status constant for each batch 42STEP_CLEARED = 0 43STEP_CALLED = 1 44 45 46def update_restructuring_status(old_status, new_status): 47 """Update restructured variable during add_validation_score 48 49 Update the restructuring status based on the new status. 50 If the new status is that there was not an update, 51 dont overwrite the old status which may show there was an update. 52 53 Parameters 54 ---------- 55 old_status : int 56 The old restructuring status. 57 new_status : int 58 The new restructuring status. 59 60 Returns 61 ------- 62 int 63 The updated restructuring status. 64 65 """ 66 if new_status == NETWORK_RESTRUCTURED or new_status == TRAINING_COMPLETE: 67 return NETWORK_RESTRUCTURED 68 else: 69 return old_status 70 71 72def update_learning_rate(): 73 """Update the learning rate in the tracker.""" 74 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 75 learning_rate = param_group["lr"] 76 GPA.pai_tracker.add_learning_rate(learning_rate) 77 78 79def update_param_count(net): 80 """Update the parameter count in the tracker if not already set. 81 82 Parameters 83 ---------- 84 net : torch.nn.Module 85 The neural network model to count parameters for. 86 Returns 87 ------- 88 None 89 """ 90 if len(GPA.pai_tracker.member_vars["param_counts"]) == 0: 91 GPA.pai_tracker.member_vars["param_counts"].append(UPA.count_params(net)) 92 93 94def check_input_problems(net, accuracy): 95 """Check for potential input problems in add_validation_score. 96 97 Parameters 98 ---------- 99 net : torch.nn.Module 100 The neural network model to check. 101 accuracy : float, int, or torch.Tensor 102 The accuracy score to validate. 103 104 Returns 105 ------- 106 float 107 The validated accuracy score. 108 109 """ 110 111 # Make sure you are passing in the model and not the dataparallel wrapper 112 if issubclass(type(net), nn.DataParallel): 113 print("Need to call .module when using add validation score") 114 pdb.set_trace() 115 sys.exit(-1) 116 117 if "module" in net.__dir__(): 118 print("Need to call .module when using add validation score") 119 pdb.set_trace() 120 sys.exit(-1) 121 122 if not isinstance(accuracy, (float, int)): 123 try: 124 accuracy = accuracy.item() 125 except: 126 print( 127 "Scores added for add_validation_score should be " 128 "float, int, or tensor, yours is a:" 129 ) 130 print(type(accuracy)) 131 pdb.set_trace() 132 sys.exit(-1) 133 return accuracy 134 135 136def update_running_accuracy(accuracy, epochs_since_cycle_switch): 137 """Add the new accuracy to the tracker. 138 139 Parameters 140 ---------- 141 accuracy : float, int, or torch.Tensor 142 The accuracy score to add. 143 epochs_since_cycle_switch : int 144 The number of epochs since the last cycle switch. 145 146 Returns 147 ------- 148 None 149 150 """ 151 # Only update running_accuracy when neurons are being updated 152 if GPA.pai_tracker.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 153 if epochs_since_cycle_switch < GPA.pc.get_initial_history_after_switches(): 154 if epochs_since_cycle_switch <= 0: 155 GPA.pai_tracker.member_vars["running_accuracy"] = accuracy 156 else: 157 GPA.pai_tracker.member_vars[ 158 "running_accuracy" 159 ] = GPA.pai_tracker.member_vars["running_accuracy"] * ( 160 1 - (1.0 / (epochs_since_cycle_switch + 1)) 161 ) + accuracy * ( 162 1.0 / (epochs_since_cycle_switch + 1) 163 ) 164 else: 165 GPA.pai_tracker.member_vars[ 166 "running_accuracy" 167 ] = GPA.pai_tracker.member_vars["running_accuracy"] * ( 168 1.0 - 1.0 / GPA.pc.get_history_lookback() 169 ) + accuracy * ( 170 1.0 / GPA.pc.get_history_lookback() 171 ) 172 173 GPA.pai_tracker.member_vars["accuracies"].append(accuracy) 174 if GPA.pai_tracker.member_vars["mode"] == "n": 175 GPA.pai_tracker.member_vars["n_accuracies"].append(accuracy) 176 177 if ( 178 GPA.pc.get_drawing_pai() 179 or GPA.pai_tracker.member_vars["mode"] == "n" 180 or GPA.pc.get_learn_dendrites_live() 181 ): 182 GPA.pai_tracker.member_vars["running_accuracies"].append( 183 GPA.pai_tracker.member_vars["running_accuracy"] 184 ) 185 186 187def score_beats_current_best(new_score, old_score): 188 """Check if the new score beats the current best score. 189 190 Parameters 191 ---------- 192 new_score : float 193 The new score to compare. 194 old_score : float 195 The old score to compare against. 196 197 Returns 198 ------- 199 bool 200 True if the new score beats the old score, False otherwise. 201 202 Notes 203 ----- 204 Must beat the old score by the margins set in globals for improvement thresholds. 205 206 """ 207 return ( 208 GPA.pai_tracker.member_vars["maximizing_score"] 209 and (new_score * (1.0 - GPA.pc.get_improvement_threshold()) > old_score) 210 and new_score - GPA.pc.get_improvement_threshold_raw() > old_score 211 ) or ( 212 (not GPA.pai_tracker.member_vars["maximizing_score"]) 213 and (new_score * (1.0 + GPA.pc.get_improvement_threshold()) < old_score) 214 and (new_score + GPA.pc.get_improvement_threshold_raw()) < old_score 215 ) 216 217 218def check_new_best(net, accuracy, epochs_since_cycle_switch): 219 """Check if the new accuracy is a new best. 220 221 Performs saves if new best score is found. 222 223 Parameters 224 ---------- 225 net : torch.nn.Module 226 The neural network model being trained. 227 accuracy : float 228 The accuracy score to check. 229 epochs_since_cycle_switch : int 230 The number of epochs since the last cycle switch. 231 232 Returns 233 ------- 234 None 235 236 """ 237 score_improved = score_beats_current_best( 238 GPA.pai_tracker.member_vars["running_accuracy"], 239 GPA.pai_tracker.member_vars["current_best_validation_score"], 240 ) 241 242 enough_time = ( 243 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 244 ) or (GPA.pai_tracker.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME) 245 246 if ( 247 score_improved 248 or GPA.pai_tracker.member_vars["current_best_validation_score"] == 0 249 ) and enough_time: 250 251 if GPA.pai_tracker.member_vars["maximizing_score"]: 252 if GPA.pc.get_verbose(): 253 print( 254 f"\n\nGot score of {accuracy:.10f} " 255 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 256 f"*{1-GPA.pc.get_improvement_threshold()}=" 257 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 - GPA.pc.get_improvement_threshold())}) ' 258 f'which is higher than {GPA.pai_tracker.member_vars["current_best_validation_score"]:.10f} ' 259 f"by {GPA.pc.get_improvement_threshold_raw()} so setting epoch to " 260 f'{GPA.pai_tracker.member_vars["num_epochs_run"]}\n\n' 261 ) 262 else: 263 if GPA.pc.get_verbose(): 264 print( 265 f"\n\nGot score of {accuracy:.10f} " 266 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 267 f"*{1+GPA.pc.get_improvement_threshold()}=" 268 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 + GPA.pc.get_improvement_threshold())}) ' 269 f'which is lower than {GPA.pai_tracker.member_vars["current_best_validation_score"]:.10f} ' 270 f'so setting epoch to {GPA.pai_tracker.member_vars["num_epochs_run"]}\n\n' 271 ) 272 273 # Set the new best score 274 GPA.pai_tracker.member_vars["current_best_validation_score"] = ( 275 GPA.pai_tracker.member_vars["running_accuracy"] 276 ) 277 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 278 GPA.pai_tracker.member_vars["num_epochs_run"] 279 ) 280 if GPA.pc.get_verbose(): 281 print( 282 f'2 epoch improved is {GPA.pai_tracker.member_vars["epoch_last_improved"]}' 283 ) 284 # Immediately update this list before saving so loading will have it correctly 285 GPA.pai_tracker.member_vars["last_improved_accuracies"].append( 286 GPA.pai_tracker.member_vars["epoch_last_improved"] 287 ) 288 # Check if global best 289 is_global_best = score_beats_current_best( 290 GPA.pai_tracker.member_vars["current_best_validation_score"], 291 GPA.pai_tracker.member_vars["global_best_validation_score"], 292 ) 293 294 if ( 295 is_global_best 296 or GPA.pai_tracker.member_vars["global_best_validation_score"] == 0 297 ): 298 if GPA.pc.get_verbose(): 299 print( 300 f"This also beats global best of " 301 f'{GPA.pai_tracker.member_vars["global_best_validation_score"]} so saving' 302 ) 303 GPA.pai_tracker.member_vars["global_best_validation_score"] = ( 304 GPA.pai_tracker.member_vars["current_best_validation_score"] 305 ) 306 GPA.pai_tracker.member_vars["current_n_set_global_best"] = True 307 UPA.save_system(net, GPA.pc.get_save_name(), "best_model") 308 if GPA.pc.get_pai_saves(): 309 UPA.pai_save_system(net, GPA.pc.get_save_name(), "best_model") 310 else: 311 if GPA.pc.get_verbose(): 312 print("Not saving new best because:") 313 if epochs_since_cycle_switch <= GPA.pc.get_initial_history_after_switches(): 314 print( 315 f"Not enough history since switch {epochs_since_cycle_switch} <= " 316 f"{GPA.pc.get_initial_history_after_switches()}" 317 ) 318 elif GPA.pai_tracker.member_vars["maximizing_score"]: 319 print( 320 f"Got score of {accuracy} " 321 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 322 f"*{1-GPA.pc.get_improvement_threshold()}=" 323 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 - GPA.pc.get_improvement_threshold())}) ' 324 f"which is not higher than " 325 f'{GPA.pai_tracker.member_vars["current_best_validation_score"]}' 326 ) 327 else: 328 print( 329 f"Got score of {accuracy} " 330 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 331 f"*{1+GPA.pc.get_improvement_threshold()}=" 332 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 + GPA.pc.get_improvement_threshold())}) ' 333 f"which is not lower than " 334 f'{GPA.pai_tracker.member_vars["current_best_validation_score"]}' 335 ) 336 GPA.pai_tracker.member_vars["last_improved_accuracies"].append( 337 GPA.pai_tracker.member_vars["epoch_last_improved"] 338 ) 339 # If it's the first epoch, save as best anyway 340 if len(GPA.pai_tracker.member_vars["accuracies"]) == 1: 341 if GPA.pc.get_verbose(): 342 print("Saving first model or all models") 343 UPA.save_system(net, GPA.pc.get_save_name(), "best_model") 344 if GPA.pc.get_pai_saves(): 345 UPA.pai_save_system(net, GPA.pc.get_save_name(), "best_model") 346 347 348def process_no_improvement(net): 349 """Handle the case where no improvement is observed. 350 351 If the new dendrite did not improve scores, but its time to switch modes 352 either trigger the end of learning or reset to the previous dendrite 353 to try again. 354 355 Parameters 356 ---------- 357 net : torch.nn.Module 358 The neural network model being trained. 359 360 Returns 361 ------- 362 int 363 The status of restructuring or training completion. 364 torch.nn.Module 365 The potentially modified neural network model. 366 367 """ 368 if GPA.pc.get_verbose(): 369 print( 370 f"Planning to switch to p mode but best beat last: " 371 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]} ' 372 f"current start lr steps: " 373 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 374 f"and last maximum lr steps: " 375 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 376 f'for rate: {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]:.8f}' 377 ) 378 379 now = datetime.now() 380 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 381 382 if GPA.pc.get_verbose(): 383 print( 384 f'1 saving break {dt_string}_noImprove_lr_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 385 ) 386 387 GPA.pai_tracker.save_graphs( 388 f'{dt_string}_noImprove_lr_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 389 ) 390 391 if ( 392 GPA.pai_tracker.member_vars["num_dendrite_tries"] 393 < GPA.pc.get_max_dendrite_tries() -1 394 ): 395 if not GPA.pc.get_silent(): 396 print( 397 f"The newest added dendrites did not improve but current tries " 398 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"] + 1} ' 399 f"is less than max tries {GPA.pc.get_max_dendrite_tries()} " 400 f"so loading last switch and trying new Dendrites." 401 ) 402 old_tries = GPA.pai_tracker.member_vars["num_dendrite_tries"] 403 # Load best model from previous n mode 404 net = UPA.change_learning_modes( 405 net, 406 GPA.pc.get_save_name(), 407 "best_model", 408 GPA.pai_tracker.member_vars["doing_pai"], 409 ) 410 GPA.pai_tracker.member_vars["num_dendrite_tries"] = old_tries + 1 411 return NETWORK_RESTRUCTURED, net 412 else: 413 if not GPA.pc.get_silent(): 414 print( 415 f"The newest added dendrites did not improve system and " 416 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"] + 1} > ' 417 f"{GPA.pc.get_max_dendrite_tries()} so returning training_complete." 418 ) 419 print( 420 "You should now exit your training loop and " 421 "best_model will be your final model for inference" 422 ) 423 if not GPA.pc.get_perforated_backpropagation() and GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 424 print("For improved results, try perforated backpropagation next time!") 425 UPA.load_system(net, GPA.pc.get_save_name(), "best_model", switch_call=True) 426 print('before graphs') 427 GPA.pai_tracker.save_graphs() 428 print('after graphs') 429 UPA.pai_save_system(net, GPA.pc.get_save_name(), "final_clean") 430 print('after save') 431 return TRAINING_COMPLETE, net 432 433 434def process_final_network(net): 435 """When the max number of dendrites has been hit load the best_model and return 436 437 Parameters 438 ---------- 439 net : torch.nn.Module 440 The neural network model being trained. 441 442 Returns 443 ------- 444 torch.nn.Module 445 The final neural network model. 446 """ 447 448 if not GPA.pc.get_silent(): 449 print( 450 f"Last Dendrites were good and this hit the max of {GPA.pc.get_max_dendrites()}" 451 ) 452 if not GPA.pc.get_perforated_backpropagation() and GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 453 print("For improved results, try perforated backpropagation next time!") 454 GPA.pai_tracker.save_graphs("before_final") 455 UPA.load_system(net, GPA.pc.get_save_name(), "best_model", switch_call=True) 456 GPA.pai_tracker.save_graphs() 457 UPA.pai_save_system(net, GPA.pc.get_save_name(), "final_clean") 458 return net 459 460 461def process_scheduler_update(net, accuracy, epochs_since_cycle_switch): 462 """Updates the scheduler 463 464 This increments the scheduler, but if we are automatically sweeping 465 to find the best initial learning rate for a new set of dendrites 466 this function also triggers the network at addition time to 467 try the next value. 468 469 Process for finding best initial learning rate for dendrites: 470 1. Start at default rate 471 2. Learn at that rate until scheduler increments twice 472 3. Save that version, start dendrites at LR current increment - 1 473 4. Repeat 2 and 3 until version has worse final score at set LR 474 5. Load previous model with best accuracy at that LR as initial rate 475 476 Parameters 477 ---------- 478 net : torch.nn.Module 479 The neural network model being trained. 480 accuracy : float 481 The accuracy of the model at the current learning rate. 482 epochs_since_cycle_switch : int 483 The number of epochs since the last cycle switch. 484 485 Returns 486 ------- 487 int 488 The status of restructuring or training completion. 489 torch.nn.Module 490 The potentially modified neural network model. 491 """ 492 493 restructured = False 494 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 495 learning_rate1 = param_group["lr"] 496 497 if ( 498 type(GPA.pai_tracker.member_vars["scheduler_instance"]) 499 is torch.optim.lr_scheduler.ReduceLROnPlateau 500 ): 501 if ( 502 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 503 or GPA.pai_tracker.member_vars["mode"] == "p" 504 ): 505 if GPA.pc.get_verbose(): 506 print( 507 f"Updating scheduler with last improved " 508 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]} ' 509 f'from current {GPA.pai_tracker.member_vars["num_epochs_run"]}' 510 ) 511 if GPA.pai_tracker.member_vars["scheduler"] is not None: 512 GPA.pai_tracker.member_vars["scheduler_instance"].step(metrics=accuracy) 513 if ( 514 GPA.pai_tracker.member_vars["scheduler"] 515 is torch.optim.lr_scheduler.ReduceLROnPlateau 516 ): 517 if GPA.pc.get_verbose(): 518 print( 519 f"Scheduler is now at " 520 f'{GPA.pai_tracker.member_vars["scheduler_instance"].num_bad_epochs} bad epochs' 521 ) 522 else: 523 if GPA.pc.get_verbose(): 524 print("Not stepping optimizer since hasnt initialized") 525 526 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 527 if ( 528 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 529 or GPA.pai_tracker.member_vars["mode"] == "p" 530 ): 531 if GPA.pc.get_verbose(): 532 if hasattr(GPA.pai_tracker.member_vars["scheduler_instance"], '_step_count'): 533 count = GPA.pai_tracker.member_vars["scheduler_instance"]._step_count 534 else: 535 count = GPA.pai_tracker.member_vars["scheduler_instance"].last_epoch 536 537 print( 538 f"Incrementing scheduler to count " 539 f'{count}' 540 ) 541 GPA.pai_tracker.member_vars["scheduler_instance"].step() 542 if ( 543 GPA.pai_tracker.member_vars["scheduler"] 544 is torch.optim.lr_scheduler.ReduceLROnPlateau 545 ): 546 if GPA.pc.get_verbose(): 547 print( 548 f"Scheduler is now at " 549 f'{GPA.pai_tracker.member_vars["scheduler_instance"].num_bad_epochs} bad epochs' 550 ) 551 552 if ( 553 epochs_since_cycle_switch <= GPA.pc.get_initial_history_after_switches() 554 and GPA.pai_tracker.member_vars["mode"] == "n" 555 ): 556 if GPA.pc.get_verbose(): 557 print( 558 f"Not stepping with history {GPA.pc.get_initial_history_after_switches()} " 559 f"and current {epochs_since_cycle_switch}" 560 ) 561 562 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 563 learning_rate2 = param_group["lr"] 564 565 stepped = False 566 at_last_count = False 567 568 if GPA.pc.get_verbose(): 569 print( 570 f"Checking if at last with scores " 571 f'{len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"])}, ' 572 f"count since switch {epochs_since_cycle_switch} " 573 f"and last total lr step count " 574 f'{GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"]}' 575 ) 576 577 # Check if at double or exactly the test count 578 if ( 579 len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 0 580 and epochs_since_cycle_switch 581 == GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] * 2 582 ) or ( 583 len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 1 584 and epochs_since_cycle_switch 585 == GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] 586 ): 587 at_last_count = True 588 589 if GPA.pc.get_verbose(): 590 print( 591 f"At last count {at_last_count} with count {epochs_since_cycle_switch} " 592 f'and last LR count {GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"]}' 593 ) 594 595 if learning_rate1 != learning_rate2: 596 stepped = True 597 GPA.pai_tracker.member_vars["current_step_count"] += 1 598 599 if GPA.pc.get_verbose(): 600 print( 601 f"Learning rate just stepped to {learning_rate2:.10e} " 602 f'with {GPA.pai_tracker.member_vars["current_step_count"]} total steps' 603 ) 604 605 if ( 606 GPA.pai_tracker.member_vars["current_step_count"] 607 == GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] 608 ): 609 if GPA.pc.get_verbose(): 610 print( 611 f'{GPA.pai_tracker.member_vars["current_step_count"]} ' 612 f"steps is the max of the last switch mode" 613 ) 614 # Set it when 1->2 gets to 2, not when 0->1 hits 2 as stopping point 615 if ( 616 GPA.pai_tracker.member_vars["current_step_count"] 617 - GPA.pai_tracker.member_vars[ 618 "current_n_learning_rate_initial_skip_steps" 619 ] 620 == 1 621 ): 622 GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] = ( 623 epochs_since_cycle_switch 624 ) 625 626 if GPA.pc.get_verbose(): 627 print( 628 f"Learning rates were {learning_rate1:.8e} and {learning_rate2:.8e} " 629 f'started with {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 630 f'and is now at {GPA.pai_tracker.member_vars["current_step_count"]} ' 631 f'committed {GPA.pai_tracker.member_vars["committed_to_initial_rate"]} ' 632 f"then either this (non zero) or eventually comparing to " 633 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 634 f'steps or rate {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]:.8f}' 635 ) 636 637 # If learning rate just stepped, check restart at lower rate 638 if ( 639 (GPA.pai_tracker.member_vars["scheduler"] is not None) 640 and 641 # If potentially might have higher accuracy 642 ( 643 (GPA.pai_tracker.member_vars["mode"] == "n") 644 or GPA.pc.get_learn_dendrites_live() 645 ) 646 and 647 # And learning rate just stepped 648 (stepped or at_last_count) 649 ): 650 651 # If this is the first dendrite addition (last_max_learning_rate_steps == 0), 652 # immediately commit to the initial rate without searching 653 if GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] == 0: 654 if GPA.pc.get_verbose(): 655 print( 656 f"First dendrite addition detected (last_max_learning_rate_steps == 0), " 657 f"immediately committing to initial rate without search" 658 ) 659 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 660 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 661 GPA.pai_tracker.member_vars["current_step_count"] 662 ) 663 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 664 learning_rate2 665 ) 666 667 # If hasn't committed to a learning rate for this cycle yet 668 if not GPA.pai_tracker.member_vars["committed_to_initial_rate"]: 669 best_score_so_far = GPA.pai_tracker.member_vars[ 670 "global_best_validation_score" 671 ] 672 673 if GPA.pc.get_verbose(): 674 print( 675 f"In statements to check next learning rate with " 676 f"stepped {stepped} and max count {at_last_count}" 677 ) 678 679 # If no scores saved for this dendrite and initial LR test did second step 680 if len( 681 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] 682 ) == 0 and ( 683 GPA.pai_tracker.member_vars["current_step_count"] 684 - GPA.pai_tracker.member_vars[ 685 "current_n_learning_rate_initial_skip_steps" 686 ] 687 == 2 688 or at_last_count 689 ): 690 691 restructured = True 692 GPA.pai_tracker.clear_optimizer_and_scheduler() 693 694 # Save system for this initial condition 695 old_global = GPA.pai_tracker.member_vars["global_best_validation_score"] 696 old_accuracy = GPA.pai_tracker.member_vars[ 697 "current_best_validation_score" 698 ] 699 old_counts = GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] 700 skip1 = GPA.pai_tracker.member_vars[ 701 "current_n_learning_rate_initial_skip_steps" 702 ] 703 704 now = datetime.now() 705 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 706 707 GPA.pai_tracker.save_graphs( 708 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 709 ) 710 711 if GPA.pc.get_test_saves(): 712 UPA.save_system( 713 net, 714 GPA.pc.get_save_name(), 715 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 716 ) 717 718 if GPA.pc.get_verbose(): 719 print( 720 f"Saving with initial steps: {dt_string}_PBCount_" 721 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 722 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 723 f"with current best {old_accuracy}" 724 ) 725 726 # Load back at start and try with lower initial learning rate 727 net = UPA.load_system( 728 net, 729 GPA.pc.get_save_name(), 730 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 731 switch_call=True, 732 ) 733 GPA.pai_tracker.member_vars[ 734 "current_n_learning_rate_initial_skip_steps" 735 ] = (skip1 + 1) 736 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"].append( 737 old_accuracy 738 ) 739 GPA.pai_tracker.member_vars["global_best_validation_score"] = old_global 740 GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] = old_counts 741 742 # If there is one score already, this is first step at next score 743 elif len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 1: 744 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"].append( 745 GPA.pai_tracker.member_vars["current_best_validation_score"] 746 ) 747 748 # If this LR's score was worse than last LR's score 749 lr_score_worse = False 750 if GPA.pai_tracker.member_vars["maximizing_score"]: 751 lr_score_worse = ( 752 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0] 753 > GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1] 754 ) 755 else: 756 lr_score_worse = ( 757 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0] 758 < GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1] 759 ) 760 761 if lr_score_worse: 762 restructured = True 763 GPA.pai_tracker.clear_optimizer_and_scheduler() 764 765 if GPA.pc.get_verbose(): 766 print( 767 f'Got initial {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1} ' 768 f'step score {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0]} ' 769 f'and {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 770 f'score at step {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1]} ' 771 f"so loading old score" 772 ) 773 774 prior_best = GPA.pai_tracker.member_vars[ 775 "current_cycle_lr_max_scores" 776 ][0] 777 778 now = datetime.now() 779 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 780 781 GPA.pai_tracker.save_graphs( 782 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 783 ) 784 785 if GPA.pc.get_test_saves(): 786 UPA.save_system( 787 net, 788 GPA.pc.get_save_name(), 789 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 790 ) 791 792 if GPA.pc.get_verbose(): 793 print( 794 f"Saving with initial steps: {dt_string}_PBCount_" 795 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 796 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 797 ) 798 799 if GPA.pc.get_test_saves(): 800 net = UPA.load_system( 801 net, 802 GPA.pc.get_save_name(), 803 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1}', 804 switch_call=True, 805 ) 806 807 # Save graphs for chosen one 808 now = datetime.now() 809 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 810 811 GPA.pai_tracker.save_graphs( 812 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}PICKED' 813 ) 814 815 if GPA.pc.get_test_saves(): 816 UPA.save_system( 817 net, 818 GPA.pc.get_save_name(), 819 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 820 ) 821 822 if GPA.pc.get_verbose(): 823 print( 824 f"Saving with initial steps: {dt_string}_PBCount_" 825 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 826 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 827 ) 828 829 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 830 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 831 GPA.pai_tracker.member_vars["current_step_count"] 832 ) 833 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 834 learning_rate2 835 ) 836 GPA.pai_tracker.member_vars["current_best_validation_score"] = ( 837 prior_best 838 ) 839 840 if GPA.pc.get_verbose(): 841 print( 842 f"Setting last max steps to " 843 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 844 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 845 ) 846 847 else: # Current LR score is better 848 if GPA.pc.get_verbose(): 849 print( 850 f'Got initial {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1} ' 851 f'step score {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0]} ' 852 f'and {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 853 f'score at step {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1]} ' 854 f"so NOT loading old score and continuing with this score" 855 ) 856 857 if at_last_count: # If this is the last one, set it to be picked 858 restructured = True 859 GPA.pai_tracker.clear_optimizer_and_scheduler() 860 861 now = datetime.now() 862 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 863 864 GPA.pai_tracker.save_graphs( 865 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}PICKED' 866 ) 867 868 if GPA.pc.get_test_saves(): 869 UPA.save_system( 870 net, 871 GPA.pc.get_save_name(), 872 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 873 ) 874 875 if GPA.pc.get_verbose(): 876 print( 877 f"Saving with initial steps: {dt_string}_PBCount_" 878 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 879 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 880 ) 881 882 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 883 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 884 GPA.pai_tracker.member_vars["current_step_count"] 885 ) 886 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 887 learning_rate2 888 ) 889 890 if GPA.pc.get_verbose(): 891 print( 892 f"Setting last max steps to " 893 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 894 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 895 ) 896 897 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 898 899 elif len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 2: 900 print( 901 "Should never be here. Please let Perforated AI know if this happened." 902 ) 903 pdb.set_trace() 904 905 GPA.pai_tracker.member_vars["global_best_validation_score"] = ( 906 best_score_so_far 907 ) 908 909 else: 910 if GPA.pc.get_verbose(): 911 print( 912 f"Setting last max steps to " 913 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 914 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 915 ) 916 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] += 1 917 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = learning_rate2 918 if restructured: 919 return NETWORK_RESTRUCTURED, net 920 else: 921 return NO_MODEL_UPDATE, net 922 923 924class PAINeuronModuleTracker: 925 """ 926 Manager class that tracks all neuron layers and dendrite layers, 927 controls when new dendrites are added, and communicates signals to modules. 928 """ 929 930 def __init__( 931 self, 932 doing_pai, 933 save_name, 934 making_graphs=True, 935 param_vals_setting=-1, 936 values_per_train_epoch=-1, 937 values_per_val_epoch=-1, 938 ): 939 """Initialize the tracker 940 941 Parameters 942 ---------- 943 doing_pai : bool 944 Whether or not dendrites should be used. 945 save_name : str 946 The base name for saving models and graphs. 947 making_graphs : bool, optional 948 Whether or not to generate graphs, by default True. 949 param_vals_setting : int, optional 950 Parameter values setting, by default -1. 951 values_per_train_epoch : int, optional 952 The number of values to look back for graphing 953 during training, by default -1 (all values). 954 values_per_val_epoch : int, optional 955 The number of values to look back for graphing 956 during validation, by default -1 (all values). 957 Returns 958 ------- 959 None 960 """ 961 962 # Dict of member vars and their types for saving 963 self.member_vars = {} 964 self.member_var_types = {} 965 966 # Whether or not PAI will be running 967 self.member_vars["doing_pai"] = doing_pai 968 self.member_var_types["doing_pai"] = "bool" 969 970 # How many Dendrites have been added 971 self.member_vars["num_dendrites_added"] = 0 972 self.member_var_types["num_dendrites_added"] = "int" 973 974 # How many Dendrites have been successfully integrated (kept) 975 self.member_vars["num_dendrites_integrated"] = 0 976 self.member_var_types["num_dendrites_integrated"] = "int" 977 978 # How many cycles have been run, *2 or *2+1 of the above 979 self.member_vars["num_cycles"] = 0 980 self.member_var_types["num_cycles"] = "int" 981 982 # Pointers to all neuron wrapped modules 983 self.neuron_module_vector = [] 984 985 # Pointers to all non neuron modules for tracking 986 self.tracked_neuron_module_vector = [] 987 988 # Neuron training or dendrite training mode 989 self.member_vars["mode"] = "n" 990 self.member_var_types["mode"] = "string" 991 992 # Number of epochs run excluding overwritten epochs 993 self.member_vars["num_epochs_run"] = -1 994 self.member_var_types["num_epochs_run"] = "int" 995 996 # Number including overwritten epochs 997 self.member_vars["total_epochs_run"] = -1 998 self.member_var_types["total_epochs_run"] = "int" 999 1000 # Last epoch that validation/correlation score was improved 1001 self.member_vars["epoch_last_improved"] = 0 1002 self.member_var_types["epoch_last_improved"] = "int" 1003 1004 # Running validation accuracy 1005 self.member_vars["running_accuracy"] = 0 1006 self.member_var_types["running_accuracy"] = "float" 1007 1008 # True if maxing validation, False if minimizing Loss 1009 self.member_vars["maximizing_score"] = True 1010 self.member_var_types["maximizing_score"] = "bool" 1011 1012 # Mode for switching back and forth between learning modes 1013 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 1014 self.member_var_types["switch_mode"] = "int" 1015 1016 # Epoch of the last switch 1017 self.member_vars["last_switch"] = 0 1018 self.member_var_types["last_switch"] = "int" 1019 1020 # Highest validation score from current cycle 1021 self.member_vars["current_best_validation_score"] = 0 1022 self.member_var_types["current_best_validation_score"] = "float" 1023 1024 # Last epoch where the learning rate was updated 1025 self.member_vars["initial_lr_test_epoch_count"] = -1 1026 self.member_var_types["initial_lr_test_epoch_count"] = "int" 1027 1028 # Highest validation score of full run 1029 self.member_vars["global_best_validation_score"] = 0 1030 self.member_var_types["global_best_validation_score"] = "float" 1031 1032 # List of switch epochs 1033 self.member_vars["switch_epochs"] = [] 1034 self.member_var_types["switch_epochs"] = "int array" 1035 1036 # Parameter counts at each network structure 1037 self.member_vars["param_counts"] = [] 1038 self.member_var_types["param_counts"] = "int array" 1039 1040 # List of epochs where switch was made to neuron training 1041 self.member_vars["n_switch_epochs"] = [] 1042 self.member_var_types["n_switch_epochs"] = "int array" 1043 1044 # List of epochs where switch was made to dendrite training 1045 self.member_vars["p_switch_epochs"] = [] 1046 self.member_var_types["p_switch_epochs"] = "int array" 1047 1048 # List of validation accuracies 1049 self.member_vars["accuracies"] = [] 1050 self.member_var_types["accuracies"] = "float array" 1051 1052 # List of epochs where score improved for scheduler updates 1053 self.member_vars["last_improved_accuracies"] = [] 1054 self.member_var_types["last_improved_accuracies"] = "int array" 1055 1056 # List of test accuracy scores registered 1057 self.member_vars["test_accuracies"] = [] 1058 self.member_var_types["test_accuracies"] = "float array" 1059 1060 # List of accuracies registered during neuron training 1061 self.member_vars["n_accuracies"] = [] 1062 self.member_var_types["n_accuracies"] = "float array" 1063 1064 # List of accuracies registered during dendrite training 1065 self.member_vars["p_accuracies"] = [] 1066 self.member_var_types["p_accuracies"] = "float array" 1067 1068 # Running average accuracies from recent epochs 1069 self.member_vars["running_accuracies"] = [] 1070 self.member_var_types["running_accuracies"] = "float array" 1071 1072 # List of additional scores recorded 1073 self.member_vars["extra_scores"] = {} 1074 self.member_var_types["extra_scores"] = "float array dictionary" 1075 1076 # Extra scores not set to be graphed 1077 self.member_vars["extra_scores_without_graphing"] = {} 1078 self.member_var_types["extra_scores_without_graphing"] = ( 1079 "float array dictionary" 1080 ) 1081 1082 # List of test scores 1083 self.member_vars["test_scores"] = [] 1084 self.member_var_types["test_scores"] = "float array" 1085 1086 # Extra scores calculated during neuron training 1087 self.member_vars["n_extra_scores"] = {} 1088 self.member_var_types["n_extra_scores"] = "float array dictionary" 1089 1090 # List of training losses calculated 1091 self.member_vars["training_loss"] = [] 1092 self.member_var_types["training_loss"] = "float array" 1093 1094 # List of learning rates at each epoch 1095 self.member_vars["training_learning_rates"] = [] 1096 self.member_var_types["training_learning_rates"] = "float array" 1097 1098 # Best dendrite scores 1099 self.member_vars["best_scores"] = [] 1100 self.member_var_types["best_scores"] = "float array array" 1101 1102 # Current dendrite scores 1103 self.member_vars["current_scores"] = [] 1104 self.member_var_types["current_scores"] = "float array array" 1105 1106 # Times for neuron training epochs 1107 self.member_vars["n_epoch_times"] = [] 1108 self.member_var_types["n_epoch_times"] = "float array" 1109 1110 # Timing values 1111 self.member_vars["p_epoch_times"] = [] 1112 self.member_var_types["p_epoch_times"] = "float array" 1113 self.member_vars["n_train_times"] = [] 1114 self.member_var_types["n_train_times"] = "float array" 1115 self.member_vars["p_train_times"] = [] 1116 self.member_var_types["p_train_times"] = "float array" 1117 self.member_vars["n_val_times"] = [] 1118 self.member_var_types["n_val_times"] = "float array" 1119 self.member_vars["p_val_times"] = [] 1120 self.member_var_types["p_val_times"] = "float array" 1121 1122 # Setting for tracking timing 1123 self.member_vars["manual_train_switch"] = False 1124 self.member_var_types["manual_train_switch"] = "bool" 1125 1126 # Tracking scores overwritten when reloading best model 1127 self.member_vars["overwritten_extras"] = [] 1128 self.member_var_types["overwritten_extras"] = "float array dictionary array" 1129 self.member_vars["overwritten_vals"] = [] 1130 self.member_var_types["overwritten_vals"] = "float array array" 1131 self.member_vars["overwritten_epochs"] = 0 1132 self.member_var_types["overwritten_epochs"] = "int" 1133 1134 # Setting for determining scores 1135 self.member_vars["param_vals_setting"] = GPA.pc.get_param_vals_setting() 1136 self.member_var_types["param_vals_setting"] = "int" 1137 1138 # Optimizer and scheduler types and instances 1139 self.member_vars["optimizer"] = None 1140 self.member_var_types["optimizer"] = "type" 1141 self.member_vars["scheduler"] = None 1142 self.member_var_types["scheduler"] = "type" 1143 self.member_vars["optimizer_instance"] = None 1144 self.member_var_types["optimizer_instance"] = "empty array" 1145 self.member_vars["scheduler_instance"] = None 1146 self.member_var_types["scheduler_instance"] = "empty array" 1147 1148 # Flag for if the tracker was loaded 1149 self.loaded = False 1150 1151 # flag for 1152 self.member_vars["step_status"] = STEP_CLEARED 1153 self.member_var_types["step_status"] = "int" 1154 1155 1156 # Settings for tracking learning rates 1157 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 1158 self.member_var_types["current_n_learning_rate_initial_skip_steps"] = "int" 1159 self.member_vars["last_max_learning_rate_steps"] = 0 1160 self.member_var_types["last_max_learning_rate_steps"] = "int" 1161 self.member_vars["last_max_learning_rate_value"] = -1 1162 self.member_var_types["last_max_learning_rate_value"] = "float" 1163 self.member_vars["current_cycle_lr_max_scores"] = [] 1164 self.member_var_types["current_cycle_lr_max_scores"] = "float array" 1165 self.member_vars["current_step_count"] = 0 1166 self.member_var_types["current_step_count"] = "int" 1167 self.member_vars["committed_to_initial_rate"] = True 1168 self.member_var_types["committed_to_initial_rate"] = "bool" 1169 self.member_vars["best_mean_score_improved_this_epoch"] = 0 1170 self.member_var_types["best_mean_score_improved_this_epoch"] = "int" 1171 1172 # Flag for if current dendrite achieved highest global score 1173 self.member_vars["current_n_set_global_best"] = True 1174 self.member_var_types["current_n_set_global_best"] = "bool" 1175 1176 # Number of tries adding this dendrite count 1177 self.member_vars["num_dendrite_tries"] = 0 1178 self.member_var_types["num_dendrite_tries"] = "int" 1179 1180 # Count of batches per epoch 1181 self.values_per_train_epoch = values_per_train_epoch 1182 self.values_per_val_epoch = values_per_val_epoch 1183 1184 self.save_name = save_name 1185 self.making_graphs = making_graphs 1186 1187 self.start_time = time.time() 1188 self.saved_time = 0 1189 self.start_epoch(internal_call=True) 1190 1191 if GPA.pc.get_verbose(): 1192 print(f'Initializing with switch_mode {self.member_vars["switch_mode"]}') 1193 1194 def to_string(self): 1195 """Convert tracker values to string for saving with safetensors.""" 1196 1197 full_string = "" 1198 for var in self.member_vars: 1199 full_string += var + "," 1200 if self.member_vars[var] is None: 1201 full_string += "None" 1202 full_string += "\n" 1203 elif self.member_var_types[var] == "bool": 1204 full_string += str(self.member_vars[var]) 1205 full_string += "\n" 1206 elif self.member_var_types[var] in ("int", "float", "string"): 1207 full_string += str(self.member_vars[var]) 1208 full_string += "\n" 1209 elif self.member_var_types[var] == "type": 1210 name = ( 1211 self.member_vars[var].__module__ 1212 + "." 1213 + self.member_vars[var].__name__ 1214 ) 1215 full_string += str(self.member_vars[var]) 1216 full_string += "\n" 1217 elif self.member_var_types[var] == "empty array": 1218 full_string += "[]" 1219 full_string += "\n" 1220 elif self.member_var_types[var] in ("int array", "float array"): 1221 full_string += "\n" 1222 string = "" 1223 for val in self.member_vars[var]: 1224 string += str(val) + "," 1225 # Remove the last comma 1226 string = string[:-1] 1227 full_string += string 1228 full_string += "\n" 1229 elif self.member_var_types[var] == "float array dictionary array": 1230 full_string += "\n" 1231 for array in self.member_vars[var]: 1232 for key in array: 1233 string = key + "," 1234 for val in array[key]: 1235 string += str(val) + "," 1236 # Remove the last comma 1237 string = string[:-1] 1238 full_string += string 1239 full_string += "\n" 1240 full_string += "endkey" 1241 full_string += "\n" 1242 full_string += "endarray" 1243 full_string += "\n" 1244 elif self.member_var_types[var] == "float array dictionary": 1245 full_string += "\n" 1246 for key in self.member_vars[var]: 1247 string = key + "," 1248 for val in self.member_vars[var][key]: 1249 string += str(val) + "," 1250 # Remove the last comma 1251 string = string[:-1] 1252 full_string += string 1253 full_string += "\n" 1254 full_string += "end" 1255 full_string += "\n" 1256 elif self.member_var_types[var] == "float array array": 1257 full_string += "\n" 1258 for array in self.member_vars[var]: 1259 string = "" 1260 for val in array: 1261 string += str(val) + "," 1262 # Remove the last comma 1263 string = string[:-1] 1264 full_string += string 1265 full_string += "\n" 1266 full_string += "end" 1267 full_string += "\n" 1268 else: 1269 print("Did not find a member variable") 1270 pdb.set_trace() 1271 return full_string 1272 1273 def from_string(self, string): 1274 """Load tracker values from string. 1275 1276 Parameters 1277 ---------- 1278 string : str 1279 The string to load from. 1280 """ 1281 f = io.StringIO(string) 1282 while True: 1283 line = f.readline() 1284 if not line: 1285 break 1286 vals = line.split(",") 1287 var = vals[0] 1288 1289 if self.member_var_types[var] == "bool": 1290 val = vals[1][:-1] 1291 if val == "True": 1292 self.member_vars[var] = True 1293 elif val == "False": 1294 self.member_vars[var] = False 1295 elif val == "1": 1296 self.member_vars[var] = 1 1297 elif val == "0": 1298 self.member_vars[var] = 0 1299 else: 1300 print("Something went wrong with loading") 1301 pdb.set_trace() 1302 elif self.member_var_types[var] == "int": 1303 val = vals[1] 1304 self.member_vars[var] = int(val) 1305 elif self.member_var_types[var] == "float": 1306 val = vals[1] 1307 self.member_vars[var] = float(val) 1308 elif self.member_var_types[var] == "string": 1309 val = vals[1][:-1] 1310 self.member_vars[var] = val 1311 elif self.member_var_types[var] == "type": 1312 # Ignore loading types, tracker should have them set up 1313 continue 1314 elif self.member_var_types[var] == "empty array": 1315 val = vals[1] 1316 self.member_vars[var] = [] 1317 elif self.member_var_types[var] == "int array": 1318 vals = f.readline()[:-1].split(",") 1319 self.member_vars[var] = [] 1320 if vals[0] == "": 1321 continue 1322 for val in vals: 1323 self.member_vars[var].append(int(val)) 1324 elif self.member_var_types[var] == "float array": 1325 vals = f.readline()[:-1].split(",") 1326 self.member_vars[var] = [] 1327 if vals[0] == "": 1328 continue 1329 for val in vals: 1330 self.member_vars[var].append(float(val)) 1331 elif self.member_var_types[var] == "float array dictionary array": 1332 self.member_vars[var] = [] 1333 line2 = f.readline()[:-1] 1334 while line2 != "endarray": 1335 temp = {} 1336 while line2 != "endkey": 1337 vals = line2.split(",") 1338 name = vals[0] 1339 temp[name] = [] 1340 vals = vals[1:] 1341 for val in vals: 1342 temp[name].append(float(val)) 1343 line2 = f.readline()[:-1] 1344 self.member_vars[var].append(temp) 1345 line2 = f.readline()[:-1] 1346 elif self.member_var_types[var] == "float array dictionary": 1347 self.member_vars[var] = {} 1348 line2 = f.readline()[:-1] 1349 while line2 != "end": 1350 vals = line2.split(",") 1351 name = vals[0] 1352 self.member_vars[var][name] = [] 1353 vals = vals[1:] 1354 for val in vals: 1355 self.member_vars[var][name].append(float(val)) 1356 line2 = f.readline()[:-1] 1357 elif self.member_var_types[var] == "float array array": 1358 self.member_vars[var] = [] 1359 line2 = f.readline()[:-1] 1360 while line2 != "end": 1361 vals = line2.split(",") 1362 self.member_vars[var].append([]) 1363 if line2: 1364 for val in vals: 1365 self.member_vars[var][-1].append(float(val)) 1366 line2 = f.readline()[:-1] 1367 else: 1368 print("Did not find a member variable") 1369 1370 pdb.set_trace() 1371 1372 def from_string_debug(self, string): 1373 """Debug function to print tracker values from string without loading them. 1374 1375 Parameters 1376 ---------- 1377 string : str 1378 The string to debug load from. 1379 """ 1380 f = io.StringIO(string) 1381 print("=== DEBUGGING TRACKER VARIABLES ===") 1382 1383 while True: 1384 line = f.readline() 1385 if not line: 1386 break 1387 vals = line.split(",") 1388 var = vals[0] 1389 1390 print(f"\nVariable: {var}") 1391 print(f"Type: {self.member_var_types.get(var, 'UNKNOWN TYPE')}") 1392 print(f"Current value: {self.member_vars.get(var, 'NOT SET')}") 1393 1394 if self.member_var_types.get(var) == "bool": 1395 val = vals[1][:-1] 1396 print(f"Would set to: {val} -> {val == 'True'}") 1397 1398 elif self.member_var_types.get(var) == "int": 1399 val = vals[1] 1400 print(f"Would set to: {int(val)}") 1401 1402 elif self.member_var_types.get(var) == "float": 1403 val = vals[1] 1404 print(f"Would set to: {float(val)}") 1405 1406 elif self.member_var_types.get(var) == "string": 1407 val = vals[1][:-1] 1408 print(f"Would set to: '{val}'") 1409 1410 elif self.member_var_types.get(var) == "type": 1411 print("Would skip (type loading)") 1412 1413 elif self.member_var_types.get(var) == "empty array": 1414 val = vals[1] 1415 print(f"Would set to: [] (empty array)") 1416 1417 elif self.member_var_types.get(var) == "int array": 1418 vals_line = f.readline()[:-1].split(",") 1419 print(f"Would set to int array with {len(vals_line)} elements:") 1420 if vals_line[0] != "": 1421 print( 1422 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1423 ) 1424 else: 1425 print(" Empty array") 1426 1427 elif self.member_var_types.get(var) == "float array": 1428 vals_line = f.readline()[:-1].split(",") 1429 print(f"Would set to float array with {len(vals_line)} elements:") 1430 if vals_line[0] != "": 1431 print( 1432 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1433 ) 1434 else: 1435 print(" Empty array") 1436 1437 elif self.member_var_types.get(var) == "float array dictionary array": 1438 print("Would process float array dictionary array:") 1439 array_count = 0 1440 line2 = f.readline()[:-1] 1441 while line2 != "endarray": 1442 key_count = 0 1443 while line2 != "endkey": 1444 vals_dict = line2.split(",") 1445 name = vals_dict[0] 1446 print( 1447 f" Array {array_count}, Key '{name}': {len(vals_dict)-1} elements" 1448 ) 1449 key_count += 1 1450 line2 = f.readline()[:-1] 1451 print(f" Array {array_count} has {key_count} keys") 1452 array_count += 1 1453 line2 = f.readline()[:-1] 1454 print(f" Total arrays: {array_count}") 1455 1456 elif self.member_var_types.get(var) == "float array dictionary": 1457 print("Would process float array dictionary:") 1458 line2 = f.readline()[:-1] 1459 key_count = 0 1460 while line2 != "end": 1461 vals_dict = line2.split(",") 1462 name = vals_dict[0] 1463 print(f" Key '{name}': {len(vals_dict)-1} elements") 1464 key_count += 1 1465 line2 = f.readline()[:-1] 1466 print(f" Total keys: {key_count}") 1467 1468 elif self.member_var_types.get(var) == "float array array": 1469 print("Would process float array array:") 1470 line2 = f.readline()[:-1] 1471 array_count = 0 1472 while line2 != "end": 1473 if line2: 1474 vals_array = line2.split(",") 1475 print(f" Array {array_count}: {len(vals_array)} elements") 1476 else: 1477 print(f" Array {array_count}: empty") 1478 array_count += 1 1479 line2 = f.readline()[:-1] 1480 print(f" Total arrays: {array_count}") 1481 1482 else: 1483 print(f"UNKNOWN TYPE: {self.member_var_types.get(var, 'NOT FOUND')}") 1484 1485 print("\n=== END DEBUG ===") 1486 1487 def save_tracker_settings(self): 1488 """Save tracker settings for DistributedDataParallel use. 1489 1490 Saves settings in save_name/array_dims.csv 1491 1492 Parameters 1493 ---------- 1494 None 1495 Returns 1496 ------- 1497 None 1498 1499 ----- 1500 Instructions for use are in API customization.md 1501 """ 1502 if not os.path.isdir(self.save_name): 1503 os.makedirs(self.save_name) 1504 f = open(self.save_name + "/array_dims.csv", "w") 1505 for layer in self.neuron_module_vector: 1506 f.write( 1507 f"{layer.name},{layer.dendrite_module.dendrite_values[0].out_channels}\n" 1508 ) 1509 f.close() 1510 if not GPA.pc.get_silent(): 1511 print("Tracker settings saved.") 1512 print("You may now delete save_tracker_settings") 1513 1514 def initialize_tracker_settings(self): 1515 """Initialize tracker settings from saved file. 1516 1517 This function loads tracker settings from a CSV file and applies them 1518 to the layers the tracker is managing. 1519 1520 Parameters 1521 ---------- 1522 None 1523 1524 Returns 1525 ------- 1526 None 1527 1528 """ 1529 1530 channels = {} 1531 if not os.path.exists(self.save_name + "/array_dims.csv"): 1532 print( 1533 "You must call save_tracker_settings before " 1534 "initialize_tracker_settings" 1535 ) 1536 print("Follow instructions in customization.md") 1537 pdb.set_trace() 1538 f = open(self.save_name + "/array_dims.csv", "r") 1539 for line in f: 1540 channels[line.split(",")[0]] = int(line.split(",")[1]) 1541 for layer in self.neuron_module_vector: 1542 layer.dendrite_module.dendrite_values[0].setup_arrays(channels[layer.name]) 1543 1544 def set_optimizer_instance(self, optimizer_instance): 1545 """Set optimizer instance directly. 1546 1547 Parameters 1548 ---------- 1549 optimizer_instance : object 1550 The optimizer instance to set. 1551 1552 Returns 1553 ------- 1554 None 1555 1556 """ 1557 1558 try: 1559 for param_group in optimizer_instance.param_groups: 1560 if ( 1561 param_group["weight_decay"] > 0 1562 and GPA.pc.get_weight_decay_accepted() is False 1563 ): 1564 print( 1565 "For PAI training it is recommended to not use " 1566 "weight decay in your optimizer" 1567 ) 1568 1569 except: 1570 pass 1571 self.member_vars["optimizer_instance"] = optimizer_instance 1572 if GPA.pc.get_perforated_backpropagation(): 1573 TPB.setup_optimizer_pb(self.member_vars["optimizer_instance"]) 1574 1575 def set_optimizer(self, optimizer): 1576 """Set optimizer type to be initialized later 1577 1578 Parameters 1579 ---------- 1580 optimizer : object 1581 The optimizer type to set. 1582 1583 Returns 1584 ------- 1585 None 1586 1587 """ 1588 self.member_vars["optimizer"] = optimizer 1589 1590 def set_scheduler(self, scheduler): 1591 """Set scheduler type to be initialized later 1592 1593 Parameters 1594 ---------- 1595 scheduler : object 1596 The scheduler type to set. 1597 1598 Returns 1599 ------- 1600 None 1601 1602 """ 1603 if scheduler is not torch.optim.lr_scheduler.ReduceLROnPlateau: 1604 if GPA.pc.get_verbose(): 1605 print("Not using ReduceLROnPlateau, this is not recommended") 1606 self.member_vars["scheduler"] = scheduler 1607 1608 def increment_scheduler(self, num_ticks, mode): 1609 """Increment the scheduler a set number of times. 1610 1611 Used for finding best initial learning rate when adding dendrites. 1612 1613 Parameters 1614 ---------- 1615 num_ticks : int 1616 The number of scheduler steps to take. 1617 mode : str 1618 The mode for stepping the scheduler. Options are: 1619 - "step_learning_rate": Step based on improved accuracy epochs 1620 - "increment_epoch_count": Step based on total epoch count 1621 1622 Returns 1623 ------- 1624 current_steps : int 1625 The number of learning rate changes that occurred. 1626 learning_rate1 : float 1627 The final learning rate after stepping. 1628 1629 """ 1630 1631 current_steps = 0 1632 current_ticker = 0 1633 1634 for param_group in GPA.pai_tracker.member_vars[ 1635 "optimizer_instance" 1636 ].param_groups: 1637 learning_rate1 = param_group["lr"] 1638 1639 if GPA.pc.get_verbose(): 1640 print("Using scheduler:") 1641 print(type(self.member_vars["scheduler_instance"])) 1642 1643 while current_ticker < num_ticks: 1644 if GPA.pc.get_verbose(): 1645 print( 1646 f"Lower start rate initial {learning_rate1} " 1647 f'stepping {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} times' 1648 ) 1649 1650 if ( 1651 type(self.member_vars["scheduler_instance"]) 1652 is torch.optim.lr_scheduler.ReduceLROnPlateau 1653 ): 1654 if mode == "step_learning_rate": 1655 # Step with counter as last improved accuracy 1656 self.member_vars["scheduler_instance"].step( 1657 metrics=self.member_vars["last_improved_accuracies"][ 1658 GPA.pai_tracker.steps_after_switch() - 1 1659 ] 1660 ) 1661 elif mode == "increment_epoch_count": 1662 # Step with improved epoch counts up to current location 1663 self.member_vars["scheduler_instance"].step( 1664 metrics=self.member_vars["last_improved_accuracies"][ 1665 -((num_ticks - 1) - current_ticker) - 1 1666 ] 1667 ) 1668 else: 1669 self.member_vars["scheduler_instance"].step() 1670 1671 for param_group in GPA.pai_tracker.member_vars[ 1672 "optimizer_instance" 1673 ].param_groups: 1674 learning_rate2 = param_group["lr"] 1675 1676 if learning_rate2 != learning_rate1: 1677 current_steps += 1 1678 learning_rate1 = learning_rate2 1679 if mode == "step_learning_rate": 1680 current_ticker += 1 1681 if GPA.pc.get_verbose(): 1682 print(f"1 step {current_steps} to {learning_rate2}") 1683 1684 if mode == "increment_epoch_count": 1685 current_ticker += 1 1686 1687 return current_steps, learning_rate1 1688 1689 def setup_optimizer(self, net, opt_args, sched_args=None, parameters=None): 1690 """Initialize the optimizer and scheduler when added. 1691 1692 Parameters 1693 ---------- 1694 net : object 1695 The neural network model. 1696 opt_args : dict 1697 The arguments for the optimizer. 1698 sched_args : dict, optional 1699 The arguments for the scheduler, by default None. 1700 1701 Returns 1702 ------- 1703 optimizer : object 1704 The initialized optimizer instance. 1705 scheduler : object, optional 1706 The initialized scheduler instance, if a scheduler was set. 1707 1708 """ 1709 if "weight_decay" in opt_args and not GPA.pc.get_weight_decay_accepted(): 1710 print( 1711 "For PAI training it is recommended to not use " 1712 "weight decay in your optimizer" 1713 ) 1714 1715 if ("model" not in opt_args.keys()) and "params" not in opt_args.keys(): 1716 print("In setup_optimizer it will be depreciated to not pass in params yourself in the future") 1717 print("please change the settings to include params") 1718 if self.member_vars["mode"] == "n": 1719 if parameters is not None: 1720 opt_args["params"] = parameters 1721 else: 1722 opt_args["params"] = filter(lambda p: p.requires_grad, net.parameters()) 1723 else: 1724 params = UPA.get_pai_network_params(net) 1725 if parameters is not None: 1726 # Filter parameters to only those in params, preserving weight_decay 1727 params_set = set(params) 1728 filtered_params = [] 1729 for param_group in parameters: 1730 filtered_group_params = [p for p in param_group["params"] if p in params_set] 1731 if filtered_group_params: 1732 filtered_params.append({ 1733 "params": filtered_group_params, 1734 "weight_decay": param_group["weight_decay"] 1735 }) 1736 opt_args["params"] = filtered_params 1737 else: 1738 opt_args["params"] = params 1739 elif "params" in opt_args.keys(): 1740 # Check if params is a list of param groups (dicts) or a single param group 1741 params_value = opt_args["params"] 1742 if isinstance(params_value, list) and len(params_value) > 0: 1743 # Check if it's a list of dicts (multiple param groups) or list of tensors (single group) 1744 if isinstance(params_value[0], dict): 1745 # Multiple param groups format: [{"params": [...], "lr": ...}, ...] 1746 # Filter each param group for requires_grad 1747 filtered_param_groups = [] 1748 for param_group in params_value: 1749 filtered_group_params = [p for p in param_group["params"] if p.requires_grad] 1750 if filtered_group_params: 1751 new_group = param_group.copy() 1752 new_group["params"] = filtered_group_params 1753 filtered_param_groups.append(new_group) 1754 opt_args["params"] = filtered_param_groups 1755 else: 1756 # Single param group format: [tensor1, tensor2, ...] or generator 1757 # Filter for requires_grad 1758 opt_args["params"] = [p for p in params_value if p.requires_grad] 1759 elif hasattr(params_value, '__iter__'): 1760 # Handle generators or other iterables 1761 opt_args["params"] = [p for p in params_value if p.requires_grad] 1762 1763 optimizer = self.member_vars["optimizer"](**opt_args) 1764 self.set_optimizer_instance(optimizer) 1765 1766 if self.member_vars["scheduler"] is not None: 1767 # Handle SequentialLR specially 1768 if self.member_vars["scheduler"] is torch.optim.lr_scheduler.SequentialLR: 1769 """ 1770 sched_args should be a dict with "schedulers" (list of tuples) and "milestones" 1771 For example: 1772 sequential_schedArgs = { 1773 "schedulers": [ 1774 (warmup_scheduler_class, warmup_schedArgs), 1775 (main_scheduler_class, main_schedArgs) 1776 ], 1777 "milestones": [switch_epoch] 1778 } 1779 """ 1780 schedulers = [] 1781 milestones = sched_args.get("milestones", []) 1782 scheduler_configs = sched_args.get("schedulers", []) 1783 1784 for scheduler_class, scheduler_args in scheduler_configs: 1785 schedulers.append(scheduler_class(optimizer, **scheduler_args)) 1786 1787 self.member_vars["scheduler_instance"] = torch.optim.lr_scheduler.SequentialLR( 1788 optimizer, schedulers=schedulers, milestones=milestones 1789 ) 1790 else: 1791 self.member_vars["scheduler_instance"] = self.member_vars["scheduler"]( 1792 optimizer, **sched_args 1793 ) 1794 current_steps = 0 1795 1796 for param_group in GPA.pai_tracker.member_vars[ 1797 "optimizer_instance" 1798 ].param_groups: 1799 learning_rate1 = param_group["lr"] 1800 1801 if GPA.pc.get_verbose(): 1802 print( 1803 f"Resetting scheduler with {GPA.pai_tracker.steps_after_switch()} " 1804 f"steps and {GPA.pc.get_initial_history_after_switches()} initial ticks to skip" 1805 ) 1806 1807 # Find setting of previously used learning rate before adding dendrites 1808 if ( 1809 GPA.pai_tracker.member_vars[ 1810 "current_n_learning_rate_initial_skip_steps" 1811 ] 1812 != 0 1813 ): 1814 additional_steps, learning_rate1 = self.increment_scheduler( 1815 GPA.pai_tracker.member_vars[ 1816 "current_n_learning_rate_initial_skip_steps" 1817 ], 1818 "step_learning_rate", 1819 ) 1820 current_steps += additional_steps 1821 1822 if self.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 1823 initial = GPA.pc.get_initial_history_after_switches() 1824 else: 1825 initial = 0 1826 1827 if GPA.pai_tracker.steps_after_switch() > initial: 1828 # Minus extra 1 because this gets called after start epoch 1829 additional_steps, learning_rate1 = self.increment_scheduler( 1830 (GPA.pai_tracker.steps_after_switch() - initial) - 1, 1831 "increment_epoch_count", 1832 ) 1833 current_steps += additional_steps 1834 1835 if GPA.pc.get_verbose(): 1836 print( 1837 f"Scheduler update loop with {current_steps} " 1838 f"ended with {learning_rate1}" 1839 ) 1840 print( 1841 f"Scheduler ended with {current_steps} steps " 1842 f"and lr of {learning_rate1}" 1843 ) 1844 1845 self.member_vars["current_step_count"] = current_steps 1846 return optimizer, self.member_vars["scheduler_instance"] 1847 else: 1848 return optimizer 1849 1850 def clear_optimizer_and_scheduler(self): 1851 """Clear the instances for saving.""" 1852 self.member_vars["optimizer_instance"] = None 1853 self.member_vars["scheduler_instance"] = None 1854 1855 def switch_time(self): 1856 """Determine if it's time to switch between neuron and dendrite training. 1857 1858 Parameters 1859 ---------- 1860 None 1861 1862 Returns 1863 ------- 1864 bool 1865 True if it's time to switch, False otherwise. 1866 1867 Notes 1868 ----- 1869 Based on current settings and history of scores. 1870 """ 1871 1872 switch_phrase = "No mode, this should never be the case." 1873 switch_number = GPA.pc.get_n_epochs_to_switch() 1874 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1875 switch_phrase = "DOING_SWITCH_EVERY_TIME" 1876 elif self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY: 1877 switch_phrase = "DOING_HISTORY" 1878 elif self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH: 1879 switch_phrase = "DOING_FIXED_SWITCH" 1880 switch_number = GPA.pc.get_fixed_switch_num() 1881 elif self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1882 switch_phrase = "DOING_NO_SWITCH" 1883 else: 1884 print( 1885 "A switch mode must be set. Check your settings for GPA.pc.set_switch_mode()." 1886 ) 1887 pdb.set_trace() 1888 if not GPA.pc.get_silent(): 1889 if(GPA.pc.get_perforated_backpropagation()): 1890 print( 1891 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1892 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1893 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1894 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1895 f'n: {switch_number}, p: {GPA.pc.get_p_epochs_to_switch()}, ' 1896 f'num_cycles: {self.member_vars["num_cycles"]}' 1897 ) 1898 else: 1899 print( 1900 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1901 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1902 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1903 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1904 f'n: {switch_number}, num_cycles: {self.member_vars["num_cycles"]}' 1905 ) 1906 print( 1907 f' Score tracking: current_n_set_global_best={self.member_vars["current_n_set_global_best"]}, ' 1908 f'global_best={self.member_vars["global_best_validation_score"]:.4f}, ' 1909 f'current_best={self.member_vars["current_best_validation_score"]:.4f}' 1910 ) 1911 if GPA.pc.get_perforated_backpropagation(): 1912 # this will fill in epoch last improved 1913 TPB.best_pai_score_improved_this_epoch(self) ## CLOSED ONLY 1914 if self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1915 if not GPA.pc.get_silent(): 1916 print("Returning False - doing no switch mode") 1917 return False 1918 1919 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1920 if not GPA.pc.get_silent(): 1921 print("Returning True - switching every time") 1922 return True 1923 1924 # Check if we're in the middle of learning rate optimization 1925 # If so, block ALL switch triggers until committed 1926 if GPA.pc.get_verbose(): 1927 print("=== LR Optimization Check ===") 1928 print(f' mode == "n": {self.member_vars["mode"] == "n"}') 1929 print(f" get_learn_dendrites_live(): {GPA.pc.get_learn_dendrites_live()}") 1930 print(f' committed_to_initial_rate: {GPA.pai_tracker.member_vars["committed_to_initial_rate"]}') 1931 print(f" get_dont_give_up_unless_learning_rate_lowered(): {GPA.pc.get_dont_give_up_unless_learning_rate_lowered()}") 1932 print(f' current_n_learning_rate_initial_skip_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"]}') 1933 print(f' last_max_learning_rate_steps: {self.member_vars["last_max_learning_rate_steps"]}') 1934 print(f' skip_steps < max_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"] < self.member_vars["last_max_learning_rate_steps"]}') 1935 print(f' scheduler is not None: {self.member_vars["scheduler"] is not None}') 1936 print("=============================") 1937 1938 if ( 1939 ((self.member_vars["mode"] == "n") or GPA.pc.get_learn_dendrites_live()) 1940 and (GPA.pai_tracker.member_vars["committed_to_initial_rate"] is False) 1941 and (GPA.pc.get_dont_give_up_unless_learning_rate_lowered()) 1942 and ( 1943 self.member_vars["current_n_learning_rate_initial_skip_steps"] 1944 <= self.member_vars["last_max_learning_rate_steps"] 1945 ) 1946 and self.member_vars["scheduler"] is not None 1947 ): 1948 if not GPA.pc.get_silent(): 1949 print( 1950 f"Returning False - learning rate optimization in progress. " 1951 f"Not committed yet. Comparing " 1952 f'initial {self.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 1953 f'to last max {self.member_vars["last_max_learning_rate_steps"]}' 1954 ) 1955 return False 1956 1957 if len(self.member_vars["switch_epochs"]) == 0: 1958 this_count = self.member_vars["num_epochs_run"] 1959 else: 1960 this_count = ( 1961 self.member_vars["num_epochs_run"] 1962 - self.member_vars["switch_epochs"][-1] 1963 ) 1964 cap_switch = False 1965 if GPA.pc.get_perforated_backpropagation(): 1966 cap_switch = TPB.check_cap_switch(self, this_count) 1967 1968 if self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY and ( 1969 ( 1970 (self.member_vars["mode"] == "n") 1971 and ( 1972 self.member_vars["num_epochs_run"] 1973 - self.member_vars["epoch_last_improved"] 1974 >= GPA.pc.get_n_epochs_to_switch() 1975 ) 1976 and this_count 1977 >= GPA.pc.get_initial_history_after_switches() 1978 + GPA.pc.get_n_epochs_to_switch() 1979 ) 1980 or (GPA.pc.get_perforated_backpropagation() and TPB.history_switch(self)) 1981 or cap_switch 1982 ): 1983 if not GPA.pc.get_silent(): 1984 print("Returning True - History and last improved is hit") 1985 return True 1986 1987 if self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH and ( 1988 ( 1989 self.member_vars["total_epochs_run"] % GPA.pc.get_fixed_switch_num() 1990 == GPA.pc.get_fixed_switch_num() - 1 1991 ) 1992 and self.member_vars["num_epochs_run"] 1993 >= GPA.pc.get_first_fixed_switch_num() - 1 1994 ): 1995 if not GPA.pc.get_silent(): 1996 print("Returning True - Fixed switch number is hit") 1997 return True 1998 1999 if not GPA.pc.get_silent(): 2000 print("Returning False - no triggers to switch have been hit") 2001 return False 2002 2003 def steps_after_switch(self): 2004 """Based on settings, return value for steps since a switch. 2005 2006 Different options for param vals setting determine what is returned. 2007 2008 Parameters 2009 ---------- 2010 None 2011 2012 Returns 2013 ------- 2014 int 2015 The number of epochs since the last switch, or total epochs run, 2016 depending on settings. 2017 2018 """ 2019 if self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_TOTAL_EPOCH: 2020 return self.member_vars["num_epochs_run"] 2021 elif ( 2022 self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_UPDATE_EPOCH 2023 ): 2024 return self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2025 elif ( 2026 self.member_vars["param_vals_setting"] 2027 == GPA.pc.PARAM_VALS_BY_NEURON_EPOCH_START 2028 ): 2029 if self.member_vars["mode"] == "p": 2030 return ( 2031 self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2032 ) 2033 else: 2034 return self.member_vars["num_epochs_run"] 2035 else: 2036 print( 2037 f'{self.member_vars["param_vals_setting"]} is not a valid param vals option' 2038 ) 2039 pdb.set_trace() 2040 2041 def add_pai_neuron_module(self, new_module, initial_add=True): 2042 """Add neuron modules to internal vectors. 2043 2044 Parameters 2045 ---------- 2046 new_module : object 2047 The new module to add. 2048 initial_add : bool, optional 2049 Whether this is the initial addition rather than loading from file 2050 2051 Returns 2052 ------- 2053 None 2054 2055 """ 2056 2057 # If it's a duplicate, ignore the second addition 2058 if new_module in self.neuron_module_vector: 2059 return 2060 self.neuron_module_vector.append(new_module) 2061 if self.member_vars["doing_pai"]: 2062 PA.set_wrapped_params(new_module) 2063 if initial_add: 2064 self.member_vars["best_scores"].append([]) 2065 self.member_vars["current_scores"].append([]) 2066 2067 def add_tracked_neuron_module(self, new_module, initial_add=True): 2068 """Add tracked modules to internal vectors 2069 2070 Parameters 2071 ---------- 2072 new_module : object 2073 The new module to add. 2074 initial_add : bool, optional 2075 Whether this is the initial addition rather than loading from file 2076 2077 Returns 2078 ------- 2079 None 2080 2081 """ 2082 # If it's a duplicate, ignore the second addition 2083 if new_module in self.tracked_neuron_module_vector: 2084 return 2085 self.tracked_neuron_module_vector.append(new_module) 2086 if self.member_vars["doing_pai"]: 2087 PA.set_tracked_params(new_module) 2088 2089 def reset_module_vector(self, net, load_from_restart): 2090 """Clear internal vectors and reset from network. 2091 2092 Parameters 2093 ---------- 2094 net : object 2095 The neural network model. 2096 load_from_restart : bool 2097 Whether loading from a restart file. 2098 2099 Returns 2100 ------- 2101 None 2102 2103 """ 2104 self.neuron_module_vector = [] 2105 self.tracked_neuron_module_vector = [] 2106 this_list = UPA.get_pai_modules(net, 0) 2107 for module in this_list: 2108 self.add_pai_neuron_module(module, initial_add=load_from_restart) 2109 this_list = UPA.get_tracked_modules(net, 0) 2110 for module in this_list: 2111 self.add_tracked_neuron_module(module, initial_add=load_from_restart) 2112 2113 def reset_vals_for_score_reset(self): 2114 """Reset cycle scores for new cycle.""" 2115 2116 if GPA.pc.get_find_best_lr(): 2117 self.member_vars["committed_to_initial_rate"] = False 2118 print("Resetting committed to initial rate to False") 2119 # If retaining all dendrties always say that the current dendrites set global best for saving and loading 2120 if GPA.pc.get_retain_all_dendrites(): 2121 self.member_vars["current_n_set_global_best"] = True 2122 self.member_vars["global_best_validation_score"] = 0 2123 else: 2124 self.member_vars["current_n_set_global_best"] = False 2125 2126 # Don't reset global best, but do reset current best 2127 self.member_vars["current_best_validation_score"] = 0 2128 self.member_vars["initial_lr_test_epoch_count"] = -1 2129 2130 def set_dendrite_training(self): 2131 """Signal all layers to start dendrite training.""" 2132 if GPA.pc.get_verbose(): 2133 print("Calling set_dendrite_training") 2134 2135 for layer in self.neuron_module_vector[:]: 2136 worked = layer.set_mode("p") 2137 """ 2138 worked is False when a layer was added to the neuron module vector 2139 but then it's never actually been used. This can happen when 2140 you have set a layer to have requires_grad = False or when 2141 you have a module as a member variable but it's not actually 2142 part of the network. Should be moved to be a tracked layer 2143 rather than a neuron layer. 2144 """ 2145 if not worked: 2146 self.neuron_module_vector.remove(layer) 2147 2148 for layer in self.tracked_neuron_module_vector[:]: 2149 worked = layer.set_mode("p") 2150 2151 self.create_new_dendrite_module() 2152 self.member_vars["mode"] = "p" 2153 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2154 2155 if GPA.pc.get_learn_dendrites_live(): 2156 self.reset_vals_for_score_reset() 2157 2158 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2159 "current_step_count" 2160 ] 2161 2162 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 2163 GPA.pai_tracker.member_vars["num_cycles"] += 1 2164 2165 2166 def set_neuron_training(self): 2167 """Signal all layers to start neuron training.""" 2168 for module in self.neuron_module_vector: 2169 module.set_mode("n") 2170 for module in self.tracked_neuron_module_vector[:]: 2171 module.set_mode("n") 2172 2173 self.member_vars["mode"] = "n" 2174 self.member_vars["num_dendrites_added"] += 1 2175 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2176 self.reset_vals_for_score_reset() 2177 2178 self.member_vars["current_cycle_lr_max_scores"] = [] 2179 if GPA.pc.get_learn_dendrites_live(): 2180 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2181 "current_step_count" 2182 ] 2183 GPA.pai_tracker.member_vars["num_cycles"] += 1 2184 2185 if GPA.pc.get_reset_best_score_on_switch(): 2186 GPA.pai_tracker.member_vars["current_best_validation_score"] = 0 2187 GPA.pai_tracker.member_vars["running_accuracy"] = 0 2188 2189 def start_epoch(self, internal_call=False): 2190 """Perform steps for when a new training epoch is about to begin. 2191 2192 Parameters 2193 ---------- 2194 internal_call : bool, optional 2195 Whether this is an internal call or manual call 2196 2197 Returns 2198 ------- 2199 None 2200 2201 Notes 2202 ----- 2203 If you ever need to call this manually, set internal_call to False. 2204 2205 """ 2206 if self.member_vars["manual_train_switch"] and internal_call: 2207 return 2208 2209 if not internal_call and not self.member_vars["manual_train_switch"]: 2210 self.member_vars["manual_train_switch"] = True 2211 self.saved_time = 0 2212 self.member_vars["num_epochs_run"] = -1 2213 self.member_vars["total_epochs_run"] = -1 2214 2215 end = time.time() 2216 if self.member_vars["manual_train_switch"]: 2217 if self.saved_time != 0: 2218 if self.member_vars["mode"] == "p": 2219 self.member_vars["p_val_times"].append(end - self.saved_time) 2220 else: 2221 self.member_vars["n_val_times"].append(end - self.saved_time) 2222 2223 if self.member_vars["mode"] == "p": 2224 for layer in self.neuron_module_vector: 2225 for m in range(0, GPA.pc.get_global_candidates()): 2226 with torch.no_grad(): 2227 if GPA.pc.get_verbose(): 2228 print(f"Resetting score for {layer.name}") 2229 # Snapshot best_score before reset so we can compute per-epoch improvement 2230 layer.dendrite_module.dendrite_values[ 2231 m 2232 ].epoch_start_best_score.copy_( 2233 layer.dendrite_module.dendrite_values[ 2234 m 2235 ].best_score.detach() 2236 ) 2237 layer.dendrite_module.dendrite_values[ 2238 m 2239 ].best_score_improved_this_epoch = ( 2240 layer.dendrite_module.dendrite_values[ 2241 m 2242 ].best_score_improved_this_epoch 2243 * 0 2244 ) 2245 layer.dendrite_module.dendrite_values[ 2246 m 2247 ].nodes_best_improved_this_epoch = ( 2248 layer.dendrite_module.dendrite_values[ 2249 m 2250 ].nodes_best_improved_this_epoch 2251 * 0 2252 ) 2253 layer.dendrite_module.dendrite_values[ 2254 m 2255 ].nodes_improved_any = ( 2256 layer.dendrite_module.dendrite_values[ 2257 m 2258 ].nodes_improved_any 2259 * 0 2260 ) 2261 if GPA.pc.get_perforated_backpropagation(): 2262 self.member_vars["best_mean_score_improved_this_epoch"] = 0 2263 self.member_vars["num_epochs_run"] += 1 2264 self.member_vars["total_epochs_run"] = ( 2265 self.member_vars["num_epochs_run"] + self.member_vars["overwritten_epochs"] 2266 ) 2267 self.saved_time = end 2268 2269 def stop_epoch(self, internal_call=False): 2270 """Perform steps when a training epoch has completed. 2271 2272 Parameters 2273 ---------- 2274 internal_call : bool, optional 2275 Whether this is an internal call or manual call 2276 2277 Returns 2278 ------- 2279 None 2280 2281 Notes 2282 ----- 2283 If you ever need to call this manually, set internal_call to False. 2284 2285 """ 2286 end = time.time() 2287 if self.member_vars["manual_train_switch"] and internal_call: 2288 return 2289 2290 if self.member_vars["manual_train_switch"]: 2291 if self.member_vars["mode"] == "p": 2292 self.member_vars["p_train_times"].append(end - self.saved_time) 2293 else: 2294 self.member_vars["n_train_times"].append(end - self.saved_time) 2295 else: 2296 if self.member_vars["mode"] == "p": 2297 self.member_vars["p_epoch_times"].append(end - self.saved_time) 2298 else: 2299 self.member_vars["n_epoch_times"].append(end - self.saved_time) 2300 2301 self.saved_time = end 2302 2303 def initialize( 2304 self, 2305 model, 2306 doing_pai=True, 2307 save_name="PAI", 2308 making_graphs=True, 2309 maximizing_score=True, 2310 num_classes=10000, 2311 values_per_train_epoch=-1, 2312 values_per_val_epoch=-1, 2313 zooming_graph=True, 2314 ): 2315 """Setup the tracker with initial settings. 2316 2317 2318 Parameters 2319 ---------- 2320 model : object 2321 The neural network model. 2322 doing_pai : bool, optional 2323 Whether to add dendrites, by default True. 2324 save_name : str, optional 2325 The name under which to save the model. 2326 making_graphs : bool, optional 2327 Whether to make graphs, by default True. 2328 maximizing_score : bool, optional 2329 Whether to maximize the score, by default True. 2330 num_classes : int, optional 2331 The number of classes in the dataset, unused 2332 values_per_train_epoch : int, optional 2333 The number of values to look back for graphing 2334 during training, by default -1 (all values). 2335 values_per_val_epoch : int, optional 2336 The number of values to look back for graphing 2337 during validation, by default -1 (all values). 2338 zooming_graph : bool, optional 2339 Whether to zoom on graphs, by default True. 2340 2341 """ 2342 model = UPA.convert_network(model) 2343 self.member_vars["doing_pai"] = doing_pai 2344 self.member_vars["maximizing_score"] = maximizing_score 2345 self.save_name = save_name 2346 self.zooming_graph = zooming_graph 2347 self.making_graphs = making_graphs 2348 2349 if not self.loaded: 2350 self.member_vars["running_accuracy"] = (1.0 / num_classes) * 100 2351 2352 self.values_per_train_epoch = values_per_train_epoch 2353 self.values_per_val_epoch = values_per_val_epoch 2354 2355 if GPA.pc.get_testing_dendrite_capacity(): 2356 if not GPA.pc.get_silent(): 2357 print("Running a test of Dendrite Capacity.") 2358 GPA.pc.set_switch_mode(GPA.pc.DOING_SWITCH_EVERY_TIME) 2359 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 2360 GPA.pc.set_retain_all_dendrites(True) 2361 GPA.pc.set_max_dendrite_tries(1000) 2362 GPA.pc.set_max_dendrites(1000) 2363 if GPA.pc.get_perforated_backpropagation(): 2364 GPA.pc.set_initial_correlation_batches(1) 2365 else: 2366 if not GPA.pc.get_silent(): 2367 print("Running Dendrite Experiment") 2368 return model 2369 2370 def generate_accuracy_plots(self, ax, save_folder, extra_string): 2371 """ 2372 Generate plots and csvs for accuracy 2373 2374 Parameters 2375 ---------- 2376 ax : object 2377 The matplotlib axis to plot on. 2378 save_folder : str 2379 The folder to save the plots and csvs in. 2380 extra_string : str 2381 An extra string to append to the filenames. 2382 2383 Returns 2384 ------- 2385 None 2386 2387 """ 2388 2389 # If scores are being saved for epochs that get overwritten, plot them 2390 for list_id in range(len(self.member_vars["overwritten_extras"])): 2391 for extra_id in self.member_vars["overwritten_extras"][list_id]: 2392 ax.plot( 2393 np.arange( 2394 len(self.member_vars["overwritten_extras"][list_id][extra_id]) 2395 ), 2396 self.member_vars["overwritten_extras"][list_id][extra_id], 2397 "r", 2398 ) 2399 ax.plot( 2400 np.arange(len(self.member_vars["overwritten_vals"][list_id])), 2401 self.member_vars["overwritten_vals"][list_id], 2402 "b", 2403 ) 2404 2405 # Determine which accuracy vector to use 2406 if GPA.pc.get_drawing_pai(): 2407 accuracies = self.member_vars["accuracies"] 2408 else: 2409 accuracies = self.member_vars["n_accuracies"] 2410 2411 # Get pointer to additional scores being saved 2412 extra_scores = self.member_vars["extra_scores"] 2413 2414 # Plot the main accuracy scores 2415 ax.plot(np.arange(len(accuracies)), accuracies, label="Validation Scores") 2416 ax.plot( 2417 np.arange(len(self.member_vars["running_accuracies"])), 2418 self.member_vars["running_accuracies"], 2419 label="Validation Running Scores", 2420 ) 2421 2422 # Plot additional scores 2423 for extra_score in extra_scores: 2424 ax.plot( 2425 np.arange(len(extra_scores[extra_score])), 2426 extra_scores[extra_score], 2427 label=extra_score, 2428 ) 2429 2430 plt.title(save_folder + "/" + self.save_name + "Scores") 2431 plt.xlabel("Epochs") 2432 plt.ylabel("Score") 2433 2434 # Add point at epoch last improved and best validation score 2435 if GPA.pc.get_drawing_pai(): 2436 ax.plot( 2437 self.member_vars["epoch_last_improved"], 2438 self.member_vars["global_best_validation_score"], 2439 "bo", 2440 label="Global best (y)", 2441 ) 2442 ax.plot( 2443 self.member_vars["epoch_last_improved"], 2444 accuracies[self.member_vars["epoch_last_improved"]], 2445 "go", 2446 label="Epoch Last Improved", 2447 ) 2448 else: 2449 if self.member_vars["mode"] == "n": 2450 missed_time = ( 2451 self.member_vars["num_epochs_run"] 2452 - self.member_vars["epoch_last_improved"] 2453 ) 2454 ax.plot( 2455 (len(self.member_vars["n_accuracies"]) - 1) - missed_time, 2456 self.member_vars["n_accuracies"][-(missed_time + 1)], 2457 "go", 2458 label="Epoch Last Improved", 2459 ) 2460 2461 # Generate csv file for the values graphed 2462 pd1 = pd.DataFrame( 2463 {"Epochs": np.arange(len(accuracies)), "Validation Scores": accuracies} 2464 ) 2465 pd2 = pd.DataFrame( 2466 { 2467 "Epochs": np.arange(len(self.member_vars["running_accuracies"])), 2468 "Validation Running Scores": self.member_vars["running_accuracies"], 2469 } 2470 ) 2471 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2472 for extra_score in extra_scores: 2473 pd2 = pd.DataFrame( 2474 { 2475 "Epochs": np.arange(len(extra_scores[extra_score])), 2476 extra_score: extra_scores[extra_score], 2477 } 2478 ) 2479 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2480 extra_scores_without_graphing = self.member_vars[ 2481 "extra_scores_without_graphing" 2482 ] 2483 for extra_score in extra_scores_without_graphing: 2484 pd2 = pd.DataFrame( 2485 { 2486 "Epochs": np.arange( 2487 len(extra_scores_without_graphing[extra_score]) 2488 ), 2489 extra_score: extra_scores_without_graphing[extra_score], 2490 } 2491 ) 2492 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2493 pd1.to_csv( 2494 save_folder + "/" + self.save_name + extra_string + "Scores.csv", 2495 index=False, 2496 ) 2497 del pd1, pd2 2498 2499 # Set y min and max to zoom in on important part of axis 2500 if ( 2501 len(self.member_vars["switch_epochs"]) > 0 2502 and self.member_vars["switch_epochs"][0] > 0 2503 and self.zooming_graph 2504 ): 2505 if GPA.pai_tracker.member_vars["maximizing_score"]: 2506 min_val = np.array( 2507 accuracies[0 : self.member_vars["switch_epochs"][0]] 2508 ).mean() 2509 for extra_score in extra_scores: 2510 min_pot = np.array( 2511 extra_scores[extra_score][ 2512 0 : self.member_vars["switch_epochs"][0] 2513 ] 2514 ).mean() 2515 if min_pot < min_val: 2516 min_val = min_pot 2517 ax.set_ylim(ymin=min_val) 2518 else: 2519 max_val = np.array( 2520 accuracies[0 : self.member_vars["switch_epochs"][0]] 2521 ).mean() 2522 for extra_score in extra_scores: 2523 max_pot = np.array( 2524 extra_scores[extra_score][ 2525 0 : self.member_vars["switch_epochs"][0] 2526 ] 2527 ).mean() 2528 if max_pot > max_val: 2529 max_val = max_pot 2530 ax.set_ylim(ymax=max_val) 2531 2532 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2533 2534 # Draw vertical lines for epochs where a dendrite switch occurred 2535 if GPA.pc.get_drawing_pai() and self.member_vars["doing_pai"]: 2536 color = "r" 2537 for switcher in self.member_vars["switch_epochs"]: 2538 plt.axvline(x=switcher, ymin=0, ymax=1, color=color) 2539 if color == "r": 2540 color = "b" 2541 else: 2542 color = "r" 2543 else: 2544 for switcher in self.member_vars["n_switch_epochs"]: 2545 plt.axvline(x=switcher, ymin=0, ymax=1, color="b") 2546 2547 def generate_time_plots(self, ax, save_folder, extra_string): 2548 """ 2549 Generate plots and csvs for timing 2550 2551 Parameters 2552 ---------- 2553 ax : object 2554 The matplotlib axis to plot on. 2555 save_folder : str 2556 The folder to save the plots and csvs in. 2557 extra_string : str 2558 An extra string to append to the filenames. 2559 2560 Returns 2561 ------- 2562 None 2563 2564 """ 2565 if self.member_vars["manual_train_switch"]: 2566 ax.plot( 2567 np.arange(len(self.member_vars["n_train_times"])), 2568 self.member_vars["n_train_times"], 2569 label="Normal Epoch Train Times", 2570 ) 2571 ax.plot( 2572 np.arange(len(self.member_vars["p_train_times"])), 2573 self.member_vars["p_train_times"], 2574 label="PAI Epoch Train Times", 2575 ) 2576 ax.plot( 2577 np.arange(len(self.member_vars["n_val_times"])), 2578 self.member_vars["n_val_times"], 2579 label="Normal Epoch Val Times", 2580 ) 2581 ax.plot( 2582 np.arange(len(self.member_vars["p_val_times"])), 2583 self.member_vars["p_val_times"], 2584 label="PAI Epoch Val Times", 2585 ) 2586 2587 plt.title( 2588 save_folder + "/" + self.save_name + "times (by train() and eval())" 2589 ) 2590 plt.xlabel("Iteration") 2591 plt.ylabel("Epoch Time in Seconds ") 2592 ax.set_ylim(ymin=0) 2593 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2594 2595 pd1 = pd.DataFrame( 2596 { 2597 "Epochs": np.arange(len(self.member_vars["n_train_times"])), 2598 "Normal Epoch Train Times": self.member_vars["n_train_times"], 2599 } 2600 ) 2601 pd2 = pd.DataFrame( 2602 { 2603 "Epochs": np.arange(len(self.member_vars["p_train_times"])), 2604 "PAI Epoch Train Times": self.member_vars["p_train_times"], 2605 } 2606 ) 2607 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2608 2609 pd2 = pd.DataFrame( 2610 { 2611 "Epochs": np.arange(len(self.member_vars["n_val_times"])), 2612 "Normal Epoch Val Times": self.member_vars["n_val_times"], 2613 } 2614 ) 2615 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2616 2617 pd2 = pd.DataFrame( 2618 { 2619 "Epochs": np.arange(len(self.member_vars["p_val_times"])), 2620 "PAI Epoch Val Times": self.member_vars["p_val_times"], 2621 } 2622 ) 2623 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2624 2625 pd1.to_csv( 2626 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2627 index=False, 2628 ) 2629 del pd1, pd2 2630 else: 2631 ax.plot( 2632 np.arange(len(self.member_vars["n_epoch_times"])), 2633 self.member_vars["n_epoch_times"], 2634 label="Normal Epoch Times", 2635 ) 2636 ax.plot( 2637 np.arange(len(self.member_vars["p_epoch_times"])), 2638 self.member_vars["p_epoch_times"], 2639 label="PAI Epoch Times", 2640 ) 2641 2642 plt.title( 2643 save_folder + "/" + self.save_name + "times (by train() and eval())" 2644 ) 2645 plt.xlabel("Iteration") 2646 plt.ylabel("Epoch Time in Seconds ") 2647 ax.set_ylim(ymin=0) 2648 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2649 2650 pd1 = pd.DataFrame( 2651 { 2652 "Epochs": np.arange(len(self.member_vars["n_epoch_times"])), 2653 "Normal Epoch Times": self.member_vars["n_epoch_times"], 2654 } 2655 ) 2656 pd2 = pd.DataFrame( 2657 { 2658 "Epochs": np.arange(len(self.member_vars["p_epoch_times"])), 2659 "PAI Epoch Times": self.member_vars["p_epoch_times"], 2660 } 2661 ) 2662 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2663 2664 pd1.to_csv( 2665 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2666 index=False, 2667 ) 2668 del pd1, pd2 2669 2670 if self.values_per_train_epoch != -1 and self.values_per_val_epoch != -1: 2671 ax2 = ax.twinx() # Second axes sharing same x-axis 2672 ax2.set_ylabel("Single Datapoint Time in Seconds") 2673 2674 ax2.plot( 2675 np.arange(len(self.member_vars["n_train_times"])), 2676 np.array(self.member_vars["n_train_times"]) 2677 / self.values_per_train_epoch, 2678 linestyle="dashed", 2679 label="Normal Train Item Times", 2680 ) 2681 ax2.plot( 2682 np.arange(len(self.member_vars["p_train_times"])), 2683 np.array(self.member_vars["p_train_times"]) 2684 / self.values_per_train_epoch, 2685 linestyle="dashed", 2686 label="PAI Train Item Times", 2687 ) 2688 ax2.plot( 2689 np.arange(len(self.member_vars["n_val_times"])), 2690 np.array(self.member_vars["n_val_times"]) / self.values_per_val_epoch, 2691 linestyle="dashed", 2692 label="Normal Val Item Times", 2693 ) 2694 ax2.plot( 2695 np.arange(len(self.member_vars["p_val_times"])), 2696 np.array(self.member_vars["p_val_times"]) / self.values_per_val_epoch, 2697 linestyle="dashed", 2698 label="PAI Val Item Times", 2699 ) 2700 ax2.tick_params(axis="y") 2701 ax2.set_ylim(ymin=0) 2702 ax2.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2703 2704 def generate_learning_rate_plots(self, ax, save_folder, extra_string): 2705 """ 2706 Generate plots and csvs for learning rate 2707 2708 Parameters 2709 ---------- 2710 ax : object 2711 The matplotlib axis to plot on. 2712 save_folder : str 2713 The folder to save the plots and csvs in. 2714 extra_string : str 2715 An extra string to append to the filenames. 2716 2717 Returns 2718 ------- 2719 None 2720 2721 """ 2722 ax.plot( 2723 np.arange(len(self.member_vars["training_learning_rates"])), 2724 self.member_vars["training_learning_rates"], 2725 label="learning_rate", 2726 ) 2727 plt.title(save_folder + "/" + self.save_name + "learning_rate") 2728 plt.xlabel("Epochs") 2729 plt.ylabel("learning_rate") 2730 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2731 2732 pd1 = pd.DataFrame( 2733 { 2734 "Epochs": np.arange(len(self.member_vars["training_learning_rates"])), 2735 "learning_rate": self.member_vars["training_learning_rates"], 2736 } 2737 ) 2738 pd1.to_csv( 2739 save_folder + "/" + self.save_name + extra_string + "learning_rate.csv", 2740 index=False, 2741 ) 2742 del pd1 2743 2744 def generate_dendrite_learning_plots(self, ax, save_folder, extra_string): 2745 """ 2746 Generate dendrite score plots for the tracker. 2747 Also saves csv files associated with the plots. 2748 """ 2749 if self.member_vars["doing_pai"]: 2750 pd1 = None 2751 pd2 = None 2752 num_colors = len(self.neuron_module_vector) 2753 2754 if ( 2755 len(self.neuron_module_vector) > 0 2756 and len(self.member_vars["current_scores"][0]) != 0 2757 ): 2758 num_colors *= 2 2759 2760 cm = plt.get_cmap("gist_rainbow") 2761 ax.set_prop_cycle( 2762 "color", [cm(1.0 * i / num_colors) for i in range(num_colors)] 2763 ) 2764 2765 for layer_id in range(len(self.neuron_module_vector)): 2766 ax.plot( 2767 np.arange(len(self.member_vars["best_scores"][layer_id])), 2768 self.member_vars["best_scores"][layer_id], 2769 label=self.neuron_module_vector[layer_id].name, 2770 ) 2771 2772 pd2 = pd.DataFrame( 2773 { 2774 "Epochs": np.arange( 2775 len(self.member_vars["best_scores"][layer_id]) 2776 ), 2777 f"Best ever for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 2778 "best_scores" 2779 ][ 2780 layer_id 2781 ], 2782 } 2783 ) 2784 2785 if pd1 is None: 2786 pd1 = pd2 2787 else: 2788 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2789 2790 if len(self.member_vars["current_scores"][layer_id]) != 0: 2791 ax.plot( 2792 np.arange(len(self.member_vars["current_scores"][layer_id])), 2793 self.member_vars["current_scores"][layer_id], 2794 label=f"Current:{self.neuron_module_vector[layer_id].name}", 2795 ) 2796 2797 pd2 = pd.DataFrame( 2798 { 2799 "Epochs": np.arange( 2800 len(self.member_vars["current_scores"][layer_id]) 2801 ), 2802 f"Best current for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 2803 "current_scores" 2804 ][ 2805 layer_id 2806 ], 2807 } 2808 ) 2809 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2810 2811 plt.title(save_folder + "/" + self.save_name + " Best PBScores") 2812 plt.xlabel("Epochs") 2813 plt.ylabel("Best PBScore") 2814 ax.legend( 2815 bbox_to_anchor=(1.05, 1), 2816 loc="upper left", 2817 ncol=max(1, math.ceil(len(self.neuron_module_vector) / 30)), 2818 ) 2819 for switcher in self.member_vars["p_switch_epochs"]: 2820 plt.axvline(x=switcher, ymin=0, ymax=1, color="r") 2821 2822 if self.member_vars["mode"] == "p": 2823 missed_time = ( 2824 self.member_vars["num_epochs_run"] 2825 - self.member_vars["epoch_last_improved"] 2826 ) 2827 plt.axvline( 2828 x=(len(self.member_vars["best_scores"][0]) - (missed_time + 1)), 2829 ymin=0, 2830 ymax=1, 2831 color="g", 2832 ) 2833 2834 # pd1 here will be none if no PB layers are created 2835 if pd1 is not None: 2836 pd1.to_csv( 2837 save_folder 2838 + "/" 2839 + self.save_name 2840 + extra_string 2841 + "Best PBScores.csv", 2842 index=False, 2843 ) 2844 del pd1, pd2 2845 2846 def generate_extra_csv_files(self, save_folder, extra_string): 2847 """ 2848 Generate additional csvs 2849 2850 Parameters 2851 ---------- 2852 save_folder : str 2853 The folder to save the plots and csvs in. 2854 extra_string : str 2855 An extra string to append to the filenames. 2856 2857 Returns 2858 ------- 2859 None 2860 2861 """ 2862 pd1 = pd.DataFrame( 2863 { 2864 "Switch Number": np.arange(len(self.member_vars["switch_epochs"])), 2865 "Switch Epoch": self.member_vars["switch_epochs"], 2866 } 2867 ) 2868 pd1.to_csv( 2869 save_folder + "/" + self.save_name + extra_string + "switch_epochs.csv", 2870 index=False, 2871 ) 2872 del pd1 2873 2874 pd1 = pd.DataFrame( 2875 { 2876 "Switch Number": np.arange(len(self.member_vars["param_counts"])), 2877 "Param Count": self.member_vars["param_counts"], 2878 } 2879 ) 2880 pd1.to_csv( 2881 save_folder + "/" + self.save_name + extra_string + "param_counts.csv", 2882 index=False, 2883 ) 2884 del pd1 2885 2886 """ 2887 Create best_arch_scores.csv file 2888 When working with dendrites there is a tradeoff between additional param count and score improvement. 2889 This file will help track that tradeoff by recording the best scores for all extra_scores 2890 and extra_scores_without_graphing for each architecture version. 2891 The scores recorded here are from the epoch when the best validation score was found 2892 within each switch_epoch boundary. 2893 """ 2894 switch_counts = len(self.member_vars["switch_epochs"]) 2895 best_valid = [] 2896 associated_params = [] 2897 2898 # Initialize dictionaries to store best scores for each extra score type 2899 best_extra_scores = {} 2900 for score_name in self.member_vars["extra_scores"]: 2901 best_extra_scores[score_name] = [] 2902 for score_name in self.member_vars["extra_scores_without_graphing"]: 2903 best_extra_scores[score_name] = [] 2904 2905 for switch in range(0, switch_counts, 2): 2906 start_index = 0 2907 if switch != 0: 2908 start_index = self.member_vars["switch_epochs"][switch - 1] + 1 2909 end_index = self.member_vars["switch_epochs"][switch] + 1 2910 2911 if GPA.pai_tracker.member_vars["maximizing_score"]: 2912 best_valid_index = start_index + np.argmax( 2913 self.member_vars["accuracies"][start_index:end_index] 2914 ) 2915 else: 2916 best_valid_index = start_index + np.argmin( 2917 self.member_vars["accuracies"][start_index:end_index] 2918 ) 2919 2920 best_valid_score = self.member_vars["accuracies"][best_valid_index] 2921 best_valid.append(best_valid_score) 2922 2923 # Get corresponding scores from all extra_scores 2924 for score_name in self.member_vars["extra_scores"]: 2925 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 2926 best_extra_scores[score_name].append( 2927 self.member_vars["extra_scores"][score_name][best_valid_index] 2928 ) 2929 else: 2930 best_extra_scores[score_name].append(None) 2931 2932 # Get corresponding scores from all extra_scores_without_graphing 2933 for score_name in self.member_vars["extra_scores_without_graphing"]: 2934 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 2935 best_extra_scores[score_name].append( 2936 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 2937 ) 2938 else: 2939 best_extra_scores[score_name].append(None) 2940 2941 if self.member_vars["doing_pai"]: 2942 associated_params.append(self.member_vars["param_counts"][switch]) 2943 else: 2944 associated_params.append(self.member_vars["param_counts"][-1]) 2945 2946 # If in neuron training mode but not the very first epoch 2947 if self.member_vars["mode"] == "n" and ( 2948 (len(self.member_vars["switch_epochs"]) == 0) 2949 or ( 2950 self.member_vars["switch_epochs"][-1] + 1 2951 != len(self.member_vars["accuracies"]) 2952 ) 2953 ): 2954 start_index = 0 2955 if len(self.member_vars["switch_epochs"]) != 0: 2956 start_index = self.member_vars["switch_epochs"][-1] + 1 2957 2958 if GPA.pai_tracker.member_vars["maximizing_score"]: 2959 best_valid_index = start_index + np.argmax( 2960 self.member_vars["accuracies"][start_index:] 2961 ) 2962 else: 2963 best_valid_index = start_index + np.argmin( 2964 self.member_vars["accuracies"][start_index:] 2965 ) 2966 2967 best_valid_score = self.member_vars["accuracies"][best_valid_index] 2968 best_valid.append(best_valid_score) 2969 2970 # Get corresponding scores from all extra_scores 2971 for score_name in self.member_vars["extra_scores"]: 2972 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 2973 best_extra_scores[score_name].append( 2974 self.member_vars["extra_scores"][score_name][best_valid_index] 2975 ) 2976 else: 2977 best_extra_scores[score_name].append(None) 2978 2979 # Get corresponding scores from all extra_scores_without_graphing 2980 for score_name in self.member_vars["extra_scores_without_graphing"]: 2981 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 2982 best_extra_scores[score_name].append( 2983 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 2984 ) 2985 else: 2986 best_extra_scores[score_name].append(None) 2987 2988 associated_params.append(self.member_vars["param_counts"][-1]) 2989 2990 # Build dataframe with all columns 2991 csv_data = { 2992 "Param Counts": associated_params, 2993 "Max Valid Scores": best_valid, 2994 } 2995 2996 # Add columns for each extra score 2997 for score_name in best_extra_scores: 2998 csv_data[score_name] = best_extra_scores[score_name] 2999 3000 pd1 = pd.DataFrame(csv_data) 3001 pd1.to_csv( 3002 save_folder + "/" + self.save_name + extra_string + "_best_arch_scores.csv", 3003 index=False, 3004 ) 3005 del pd1 3006 3007 def save_graphs(self, extra_string=""): 3008 """ 3009 Save graphs and csvs for all the values the tracker records 3010 3011 Parameters 3012 ---------- 3013 extra_string : str 3014 An extra string to append to the filenames. 3015 3016 Returns 3017 ------- 3018 None 3019 3020 """ 3021 # If running DDP only save with rank 0 3022 if "RANK" in os.environ: 3023 if int(os.environ["RANK"]) != 0: 3024 return 3025 if not self.making_graphs: 3026 return 3027 3028 save_folder = "./" + self.save_name + "/" 3029 3030 plt.ioff() 3031 fig = plt.figure(figsize=(28, 14)) 3032 3033 # Plot with accuracy scores 3034 ax = plt.subplot(221) 3035 self.generate_accuracy_plots(ax, save_folder, extra_string) 3036 3037 # Plot dendrite learning scores 3038 ax = plt.subplot(222) 3039 self.generate_dendrite_learning_plots(ax, save_folder, extra_string) 3040 3041 if GPA.pc.get_drawing_extra_graphs(): 3042 # Plot learning rates for each training epoch 3043 ax = plt.subplot(223) 3044 self.generate_learning_rate_plots(ax, save_folder, extra_string) 3045 3046 # Plot the times for each training epoch 3047 ax = plt.subplot(224) 3048 self.generate_time_plots(ax, save_folder, extra_string) 3049 3050 # Generate extra CSV files 3051 self.generate_extra_csv_files(save_folder, extra_string) 3052 3053 fig.tight_layout() 3054 plt.savefig(save_folder + "/" + self.save_name + extra_string + ".png") 3055 plt.close("all") 3056 3057 def add_loss(self, loss): 3058 """Add loss to tracking vectors. 3059 3060 Parameters 3061 ---------- 3062 loss : float or int 3063 The loss value to add. 3064 3065 Returns 3066 ------- 3067 None 3068 3069 """ 3070 if not isinstance(loss, (float, int)): 3071 loss = loss.item() 3072 self.member_vars["training_loss"].append(loss) 3073 3074 def add_learning_rate(self, learning_rate): 3075 """Add learning rate to tracking vectors. 3076 3077 Parameters 3078 ---------- 3079 learning_rate : float or int 3080 The learning rate value to add. 3081 3082 Returns 3083 ------- 3084 None 3085 3086 """ 3087 if not isinstance(learning_rate, (float, int)): 3088 learning_rate = learning_rate.item() 3089 self.member_vars["training_learning_rates"].append(learning_rate) 3090 3091 def add_extra_score(self, score, extra_score_name): 3092 """Add extra score to tracking vectors. 3093 3094 Parameters 3095 ---------- 3096 score : float or int 3097 The score value to add. 3098 3099 extra_score_name : str 3100 The name of the extra score. 3101 3102 Returns 3103 ------- 3104 None 3105 3106 """ 3107 if not isinstance(score, (float, int)): 3108 try: 3109 score = score.item() 3110 except: 3111 print( 3112 "Scores added for Perforated Backpropagation should be " 3113 "float, int, or tensor, yours is a:" 3114 ) 3115 print(type(score)) 3116 pdb.set_trace() 3117 3118 if GPA.pc.get_verbose(): 3119 print(f"Adding extra score {extra_score_name} of {float(score)}") 3120 3121 if extra_score_name not in self.member_vars["extra_scores"]: 3122 self.member_vars["extra_scores"][extra_score_name] = [] 3123 self.member_vars["extra_scores"][extra_score_name].append(score) 3124 3125 if self.member_vars["mode"] == "n": 3126 if extra_score_name not in self.member_vars["n_extra_scores"]: 3127 self.member_vars["n_extra_scores"][extra_score_name] = [] 3128 self.member_vars["n_extra_scores"][extra_score_name].append(score) 3129 3130 def add_extra_score_without_graphing(self, score, extra_score_name): 3131 """Add extra score without graphing to tracking vectors. 3132 3133 Parameters 3134 ---------- 3135 score : float or int 3136 The score value to add. 3137 3138 extra_score_name : str 3139 The name of the extra score. 3140 3141 Returns 3142 ------- 3143 None 3144 3145 """ 3146 if not isinstance(score, (float, int)): 3147 try: 3148 score = score.item() 3149 except: 3150 print( 3151 "Scores added for Perforated Backpropagation should be " 3152 "float, int, or tensor, yours is a:" 3153 ) 3154 print(type(score)) 3155 print("in add_extra_score_without_graphing") 3156 pdb.set_trace() 3157 3158 if GPA.pc.get_verbose(): 3159 print(f"Adding extra score {extra_score_name} of {float(score)}") 3160 3161 if extra_score_name not in self.member_vars["extra_scores_without_graphing"]: 3162 self.member_vars["extra_scores_without_graphing"][extra_score_name] = [] 3163 self.member_vars["extra_scores_without_graphing"][extra_score_name].append( 3164 score 3165 ) 3166 3167 def add_test_score(self, score, extra_score_name): 3168 """Add test score to tracking vectors. 3169 3170 Parameters 3171 ---------- 3172 score : float or int 3173 The score value to add. 3174 3175 extra_score_name : str 3176 The name of the extra score. 3177 3178 Returns 3179 ------- 3180 None 3181 3182 Notes 3183 ----- 3184 This function is a wrapper around `add_extra_score` that separates 3185 test score for adding to best_arch_scores.csv. 3186 3187 """ 3188 self.add_extra_score(score, extra_score_name) 3189 3190 if not isinstance(score, (float, int)): 3191 try: 3192 score = score.item() 3193 except: 3194 print( 3195 "Scores added for Perforated Backpropagation should be " 3196 "float, int, or tensor, yours is a:" 3197 ) 3198 print(type(score)) 3199 print("in add_test_score") 3200 pdb.set_trace() 3201 3202 if GPA.pc.get_verbose(): 3203 print(f"Adding test score {extra_score_name} of {float(score)}") 3204 self.member_vars["test_scores"].append(score) 3205 3206 def add_validation_score(self, accuracy, net, force_switch=False): 3207 """Function to add the validation score. 3208 3209 This is complex because it determines neuron and dendrite switching. 3210 3211 Parameters 3212 ---------- 3213 accuracy : float or int 3214 The accuracy or loss value to add. 3215 net : object 3216 The neural network model. 3217 force_switch : bool, optional 3218 Whether to force a switch, by default False. 3219 3220 Returns 3221 ------- 3222 net : object 3223 The potentially modified neural network model. 3224 training_complete : bool 3225 Whether training is complete. 3226 restructured : bool 3227 Whether the model has been restructured. 3228 3229 Notes 3230 ----- 3231 WARNING: Do not call self anywhere in this function. When systems 3232 get loaded the actual tracker you are working with can change. 3233 """ 3234 3235 if not GPA.pc.get_silent(): 3236 print(f"Adding validation score {accuracy:.8f}") 3237 3238 update_learning_rate() 3239 update_param_count(net) 3240 3241 accuracy = check_input_problems(net, accuracy) 3242 3243 if len(GPA.pai_tracker.member_vars["switch_epochs"]) == 0: 3244 epochs_since_cycle_switch = GPA.pai_tracker.member_vars["num_epochs_run"] 3245 else: 3246 epochs_since_cycle_switch = ( 3247 GPA.pai_tracker.member_vars["num_epochs_run"] 3248 - GPA.pai_tracker.member_vars["switch_epochs"][-1] 3249 ) 3250 3251 update_running_accuracy(accuracy, epochs_since_cycle_switch) 3252 if GPA.pc.get_perforated_backpropagation(): 3253 TPB.update_pb_scores(self) 3254 3255 GPA.pai_tracker.stop_epoch(internal_call=True) 3256 3257 # If it is neuron training mode 3258 if ( 3259 GPA.pai_tracker.member_vars["mode"] == "n" 3260 or GPA.pc.get_learn_dendrites_live() 3261 ): 3262 check_new_best(net, accuracy, epochs_since_cycle_switch) 3263 elif GPA.pc.get_perforated_backpropagation(): 3264 TPB.check_best_pai_score_improvement() 3265 3266 # Save the latest model 3267 if GPA.pc.get_test_saves(): 3268 UPA.save_system(net, GPA.pc.get_save_name(), "latest") 3269 if GPA.pc.get_pai_saves(): 3270 UPA.pai_save_system(net, GPA.pc.get_save_name(), "latest") 3271 3272 restructuring_status_value = NO_MODEL_UPDATE 3273 # If it is time to switch based on scores and counter or a manual switch 3274 if GPA.pai_tracker.switch_time() or force_switch: 3275 # If testing dendrite capacity switch after enough dendrites added 3276 if ( 3277 (GPA.pai_tracker.member_vars["mode"] == "n") 3278 and (GPA.pai_tracker.member_vars["num_dendrites_added"] > 2) 3279 and GPA.pc.get_testing_dendrite_capacity() 3280 ): 3281 GPA.pai_tracker.save_graphs() 3282 print( 3283 "Successfully added 3 dendrites with " 3284 "GPA.pc.set_testing_dendrite_capacity(True) (default). " 3285 "You may now set that to False and run a real experiment." 3286 ) 3287 return net, False, True 3288 3289 # If doing neuron training but this dendrite count didn't improve 3290 if ( 3291 (GPA.pai_tracker.member_vars["mode"] == "n") 3292 or GPA.pc.get_learn_dendrites_live() 3293 ) and (GPA.pai_tracker.member_vars["current_n_set_global_best"] is False): 3294 new_restructuring_status_value, net = process_no_improvement(net) 3295 # if this was the final try return that training is complete 3296 if new_restructuring_status_value == TRAINING_COMPLETE: 3297 return net, True, True 3298 else: 3299 restructuring_status_value = update_restructuring_status( 3300 restructuring_status_value, new_restructuring_status_value 3301 ) 3302 # Else if did improve, do a normal switch process 3303 else: 3304 if GPA.pc.get_verbose(): 3305 print( 3306 f"Calling switch_mode with " 3307 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]}, ' 3308 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 3309 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]}, ' 3310 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_value"]},' 3311 f'{GPA.pc.get_max_dendrites()},' 3312 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]},' 3313 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"]},' 3314 ) 3315 import pdb; pdb.set_trace 3316 # If the max number of dendrites has been hit or not doing pai and adding dendtites 3317 # then return rather than adding more 3318 if ( 3319 (GPA.pai_tracker.member_vars["mode"] == "n") 3320 and ( 3321 GPA.pc.get_max_dendrites() 3322 == GPA.pai_tracker.member_vars["num_dendrites_added"] 3323 ) 3324 ) or (GPA.pai_tracker.member_vars["doing_pai"] is False): 3325 if GPA.pc.get_verbose(): 3326 print( 3327 "Max dendrites reached or not doing PAI, finishing training" 3328 ) 3329 net = process_final_network(net) 3330 # Increment integrated if we have dendrites (means they're integrated) 3331 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3332 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3333 if not GPA.pc.get_silent(): 3334 print(f"Final dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3335 return net, True, True 3336 3337 # Otherwise if its neuron training mode reset the counter of failed dendrites 3338 # Check if we should increment integrated count BEFORE change_learning_modes loads old state 3339 should_increment_integrated = False 3340 if GPA.pai_tracker.member_vars["mode"] == "n": 3341 GPA.pai_tracker.member_vars["num_dendrite_tries"] = 0 3342 if GPA.pc.get_verbose(): 3343 print( 3344 "Adding new dendrites without resetting which means " 3345 "the last ones improved. Resetting num_dendrite_tries" 3346 ) 3347 # Remember to increment after change_learning_modes (which loads old tracker state) 3348 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3349 should_increment_integrated = True 3350 3351 GPA.pai_tracker.save_graphs( 3352 f'_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}' 3353 ) 3354 3355 if GPA.pc.get_test_saves(): 3356 UPA.save_system( 3357 net, 3358 GPA.pc.get_save_name(), 3359 f'beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3360 ) 3361 # Copy current best model from this set of dendrites 3362 # If running DDP only copy with rank 0 3363 if "RANK" not in os.environ or int(os.environ["RANK"]) == 0: 3364 shutil.copyfile( 3365 f"{GPA.pc.get_save_name()}/best_model.pt", 3366 f'{GPA.pc.get_save_name()}/best_model_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}.pt', 3367 ) 3368 3369 net = UPA.change_learning_modes( 3370 net, 3371 GPA.pc.get_save_name(), 3372 "best_model", 3373 GPA.pai_tracker.member_vars["doing_pai"], 3374 ) 3375 restructuring_status_value = NETWORK_RESTRUCTURED 3376 3377 # Now increment after change_learning_modes has loaded the best model 3378 # This ensures the increment persists and doesn't get overwritten 3379 if should_increment_integrated: 3380 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3381 if not GPA.pc.get_silent(): 3382 print(f"Dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3383 3384 # If restructured is true, clear scheduler/optimizer before saving 3385 if restructuring_status_value != NETWORK_RESTRUCTURED: 3386 print( 3387 "Restructured should always be triggered here, let us know if you encounter this situation" 3388 ) 3389 pdb.set_trace() 3390 3391 # Since there is a restructuring optimizer and scheduler must be reinitialized after return 3392 GPA.pai_tracker.clear_optimizer_and_scheduler() 3393 3394 # Save the model from after the switch 3395 UPA.save_system( 3396 net, 3397 GPA.pc.get_save_name(), 3398 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3399 ) 3400 3401 # If not time to switch and you have a scheduler, perform the update step 3402 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 3403 new_restructuring_status_value, net = process_scheduler_update( 3404 net, accuracy, epochs_since_cycle_switch 3405 ) 3406 restructuring_status_value = update_restructuring_status( 3407 restructuring_status_value, new_restructuring_status_value 3408 ) 3409 3410 GPA.pai_tracker.start_epoch(internal_call=True) 3411 GPA.pai_tracker.save_graphs() 3412 3413 if restructuring_status_value == NETWORK_RESTRUCTURED: 3414 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 3415 GPA.pai_tracker.member_vars["num_epochs_run"] 3416 ) 3417 if GPA.pc.get_verbose(): 3418 print( 3419 f"Setting epoch last improved to " 3420 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]}' 3421 ) 3422 3423 now = datetime.now() 3424 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 3425 3426 if GPA.pc.get_verbose(): 3427 print("Not saving restructure right now") 3428 3429 """ 3430 This block of code helped with a save issue with safetensors and huggingface, but it breaks DDP. 3431 Temporarily removing it to avoid DDP issues, but if you encounter save issues try adding it back in. 3432 for param in net.parameters(): 3433 param.data = param.data.contiguous() 3434 """ 3435 if GPA.pc.get_verbose(): 3436 print( 3437 f"Completed adding score. Restructured is {restructuring_status_value}, " 3438 f"\ncurrent switch list is:" 3439 ) 3440 print(GPA.pai_tracker.member_vars["switch_epochs"]) 3441 3442 # Always False for training complete if nothing triggered that training is over 3443 return net, restructuring_status_value, False 3444 3445 def clear_all_processors(self): 3446 """Clear all processors from modules.""" 3447 for module in self.neuron_module_vector: 3448 module.clear_processors() 3449 3450 def create_new_dendrite_module(self): 3451 """Add dendrite module to all neuron modules.""" 3452 for module in self.neuron_module_vector: 3453 module.create_new_dendrite_module() 3454 3455 def apply_pb_grads(self): 3456 """Apply perforated backpropagation gradients to all modules.""" 3457 if self.member_vars["mode"] == "p": 3458 for module in self.neuron_module_vector: 3459 module.apply_pb_grads() 3460 3461 def apply_pb_zero(self): 3462 """Apply perforated backpropagation zero gradients to all modules.""" 3463 if self.member_vars["mode"] == "p": 3464 for module in self.neuron_module_vector: 3465 module.apply_pb_zero()
NO_MODEL_UPDATE =
0
NETWORK_RESTRUCTURED =
1
TRAINING_COMPLETE =
2
STEP_CLEARED =
0
STEP_CALLED =
1
def
update_restructuring_status(old_status, new_status):
47def update_restructuring_status(old_status, new_status): 48 """Update restructured variable during add_validation_score 49 50 Update the restructuring status based on the new status. 51 If the new status is that there was not an update, 52 dont overwrite the old status which may show there was an update. 53 54 Parameters 55 ---------- 56 old_status : int 57 The old restructuring status. 58 new_status : int 59 The new restructuring status. 60 61 Returns 62 ------- 63 int 64 The updated restructuring status. 65 66 """ 67 if new_status == NETWORK_RESTRUCTURED or new_status == TRAINING_COMPLETE: 68 return NETWORK_RESTRUCTURED 69 else: 70 return old_status
Update restructured variable during add_validation_score
Update the restructuring status based on the new status. If the new status is that there was not an update, dont overwrite the old status which may show there was an update.
Parameters
- old_status (int): The old restructuring status.
- new_status (int): The new restructuring status.
Returns
- int: The updated restructuring status.
def
update_learning_rate():
73def update_learning_rate(): 74 """Update the learning rate in the tracker.""" 75 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 76 learning_rate = param_group["lr"] 77 GPA.pai_tracker.add_learning_rate(learning_rate)
Update the learning rate in the tracker.
def
update_param_count(net):
80def update_param_count(net): 81 """Update the parameter count in the tracker if not already set. 82 83 Parameters 84 ---------- 85 net : torch.nn.Module 86 The neural network model to count parameters for. 87 Returns 88 ------- 89 None 90 """ 91 if len(GPA.pai_tracker.member_vars["param_counts"]) == 0: 92 GPA.pai_tracker.member_vars["param_counts"].append(UPA.count_params(net))
Update the parameter count in the tracker if not already set.
Parameters
- net (torch.nn.Module): The neural network model to count parameters for.
Returns
- None
def
check_input_problems(net, accuracy):
95def check_input_problems(net, accuracy): 96 """Check for potential input problems in add_validation_score. 97 98 Parameters 99 ---------- 100 net : torch.nn.Module 101 The neural network model to check. 102 accuracy : float, int, or torch.Tensor 103 The accuracy score to validate. 104 105 Returns 106 ------- 107 float 108 The validated accuracy score. 109 110 """ 111 112 # Make sure you are passing in the model and not the dataparallel wrapper 113 if issubclass(type(net), nn.DataParallel): 114 print("Need to call .module when using add validation score") 115 pdb.set_trace() 116 sys.exit(-1) 117 118 if "module" in net.__dir__(): 119 print("Need to call .module when using add validation score") 120 pdb.set_trace() 121 sys.exit(-1) 122 123 if not isinstance(accuracy, (float, int)): 124 try: 125 accuracy = accuracy.item() 126 except: 127 print( 128 "Scores added for add_validation_score should be " 129 "float, int, or tensor, yours is a:" 130 ) 131 print(type(accuracy)) 132 pdb.set_trace() 133 sys.exit(-1) 134 return accuracy
Check for potential input problems in add_validation_score.
Parameters
- net (torch.nn.Module): The neural network model to check.
- accuracy (float, int, or torch.Tensor): The accuracy score to validate.
Returns
- float: The validated accuracy score.
def
update_running_accuracy(accuracy, epochs_since_cycle_switch):
137def update_running_accuracy(accuracy, epochs_since_cycle_switch): 138 """Add the new accuracy to the tracker. 139 140 Parameters 141 ---------- 142 accuracy : float, int, or torch.Tensor 143 The accuracy score to add. 144 epochs_since_cycle_switch : int 145 The number of epochs since the last cycle switch. 146 147 Returns 148 ------- 149 None 150 151 """ 152 # Only update running_accuracy when neurons are being updated 153 if GPA.pai_tracker.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 154 if epochs_since_cycle_switch < GPA.pc.get_initial_history_after_switches(): 155 if epochs_since_cycle_switch <= 0: 156 GPA.pai_tracker.member_vars["running_accuracy"] = accuracy 157 else: 158 GPA.pai_tracker.member_vars[ 159 "running_accuracy" 160 ] = GPA.pai_tracker.member_vars["running_accuracy"] * ( 161 1 - (1.0 / (epochs_since_cycle_switch + 1)) 162 ) + accuracy * ( 163 1.0 / (epochs_since_cycle_switch + 1) 164 ) 165 else: 166 GPA.pai_tracker.member_vars[ 167 "running_accuracy" 168 ] = GPA.pai_tracker.member_vars["running_accuracy"] * ( 169 1.0 - 1.0 / GPA.pc.get_history_lookback() 170 ) + accuracy * ( 171 1.0 / GPA.pc.get_history_lookback() 172 ) 173 174 GPA.pai_tracker.member_vars["accuracies"].append(accuracy) 175 if GPA.pai_tracker.member_vars["mode"] == "n": 176 GPA.pai_tracker.member_vars["n_accuracies"].append(accuracy) 177 178 if ( 179 GPA.pc.get_drawing_pai() 180 or GPA.pai_tracker.member_vars["mode"] == "n" 181 or GPA.pc.get_learn_dendrites_live() 182 ): 183 GPA.pai_tracker.member_vars["running_accuracies"].append( 184 GPA.pai_tracker.member_vars["running_accuracy"] 185 )
Add the new accuracy to the tracker.
Parameters
- accuracy (float, int, or torch.Tensor): The accuracy score to add.
- epochs_since_cycle_switch (int): The number of epochs since the last cycle switch.
Returns
- None
def
score_beats_current_best(new_score, old_score):
188def score_beats_current_best(new_score, old_score): 189 """Check if the new score beats the current best score. 190 191 Parameters 192 ---------- 193 new_score : float 194 The new score to compare. 195 old_score : float 196 The old score to compare against. 197 198 Returns 199 ------- 200 bool 201 True if the new score beats the old score, False otherwise. 202 203 Notes 204 ----- 205 Must beat the old score by the margins set in globals for improvement thresholds. 206 207 """ 208 return ( 209 GPA.pai_tracker.member_vars["maximizing_score"] 210 and (new_score * (1.0 - GPA.pc.get_improvement_threshold()) > old_score) 211 and new_score - GPA.pc.get_improvement_threshold_raw() > old_score 212 ) or ( 213 (not GPA.pai_tracker.member_vars["maximizing_score"]) 214 and (new_score * (1.0 + GPA.pc.get_improvement_threshold()) < old_score) 215 and (new_score + GPA.pc.get_improvement_threshold_raw()) < old_score 216 )
Check if the new score beats the current best score.
Parameters
- new_score (float): The new score to compare.
- old_score (float): The old score to compare against.
Returns
- bool: True if the new score beats the old score, False otherwise.
Notes
Must beat the old score by the margins set in globals for improvement thresholds.
def
check_new_best(net, accuracy, epochs_since_cycle_switch):
219def check_new_best(net, accuracy, epochs_since_cycle_switch): 220 """Check if the new accuracy is a new best. 221 222 Performs saves if new best score is found. 223 224 Parameters 225 ---------- 226 net : torch.nn.Module 227 The neural network model being trained. 228 accuracy : float 229 The accuracy score to check. 230 epochs_since_cycle_switch : int 231 The number of epochs since the last cycle switch. 232 233 Returns 234 ------- 235 None 236 237 """ 238 score_improved = score_beats_current_best( 239 GPA.pai_tracker.member_vars["running_accuracy"], 240 GPA.pai_tracker.member_vars["current_best_validation_score"], 241 ) 242 243 enough_time = ( 244 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 245 ) or (GPA.pai_tracker.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME) 246 247 if ( 248 score_improved 249 or GPA.pai_tracker.member_vars["current_best_validation_score"] == 0 250 ) and enough_time: 251 252 if GPA.pai_tracker.member_vars["maximizing_score"]: 253 if GPA.pc.get_verbose(): 254 print( 255 f"\n\nGot score of {accuracy:.10f} " 256 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 257 f"*{1-GPA.pc.get_improvement_threshold()}=" 258 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 - GPA.pc.get_improvement_threshold())}) ' 259 f'which is higher than {GPA.pai_tracker.member_vars["current_best_validation_score"]:.10f} ' 260 f"by {GPA.pc.get_improvement_threshold_raw()} so setting epoch to " 261 f'{GPA.pai_tracker.member_vars["num_epochs_run"]}\n\n' 262 ) 263 else: 264 if GPA.pc.get_verbose(): 265 print( 266 f"\n\nGot score of {accuracy:.10f} " 267 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 268 f"*{1+GPA.pc.get_improvement_threshold()}=" 269 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 + GPA.pc.get_improvement_threshold())}) ' 270 f'which is lower than {GPA.pai_tracker.member_vars["current_best_validation_score"]:.10f} ' 271 f'so setting epoch to {GPA.pai_tracker.member_vars["num_epochs_run"]}\n\n' 272 ) 273 274 # Set the new best score 275 GPA.pai_tracker.member_vars["current_best_validation_score"] = ( 276 GPA.pai_tracker.member_vars["running_accuracy"] 277 ) 278 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 279 GPA.pai_tracker.member_vars["num_epochs_run"] 280 ) 281 if GPA.pc.get_verbose(): 282 print( 283 f'2 epoch improved is {GPA.pai_tracker.member_vars["epoch_last_improved"]}' 284 ) 285 # Immediately update this list before saving so loading will have it correctly 286 GPA.pai_tracker.member_vars["last_improved_accuracies"].append( 287 GPA.pai_tracker.member_vars["epoch_last_improved"] 288 ) 289 # Check if global best 290 is_global_best = score_beats_current_best( 291 GPA.pai_tracker.member_vars["current_best_validation_score"], 292 GPA.pai_tracker.member_vars["global_best_validation_score"], 293 ) 294 295 if ( 296 is_global_best 297 or GPA.pai_tracker.member_vars["global_best_validation_score"] == 0 298 ): 299 if GPA.pc.get_verbose(): 300 print( 301 f"This also beats global best of " 302 f'{GPA.pai_tracker.member_vars["global_best_validation_score"]} so saving' 303 ) 304 GPA.pai_tracker.member_vars["global_best_validation_score"] = ( 305 GPA.pai_tracker.member_vars["current_best_validation_score"] 306 ) 307 GPA.pai_tracker.member_vars["current_n_set_global_best"] = True 308 UPA.save_system(net, GPA.pc.get_save_name(), "best_model") 309 if GPA.pc.get_pai_saves(): 310 UPA.pai_save_system(net, GPA.pc.get_save_name(), "best_model") 311 else: 312 if GPA.pc.get_verbose(): 313 print("Not saving new best because:") 314 if epochs_since_cycle_switch <= GPA.pc.get_initial_history_after_switches(): 315 print( 316 f"Not enough history since switch {epochs_since_cycle_switch} <= " 317 f"{GPA.pc.get_initial_history_after_switches()}" 318 ) 319 elif GPA.pai_tracker.member_vars["maximizing_score"]: 320 print( 321 f"Got score of {accuracy} " 322 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 323 f"*{1-GPA.pc.get_improvement_threshold()}=" 324 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 - GPA.pc.get_improvement_threshold())}) ' 325 f"which is not higher than " 326 f'{GPA.pai_tracker.member_vars["current_best_validation_score"]}' 327 ) 328 else: 329 print( 330 f"Got score of {accuracy} " 331 f'(average {GPA.pai_tracker.member_vars["running_accuracy"]}, ' 332 f"*{1+GPA.pc.get_improvement_threshold()}=" 333 f'{GPA.pai_tracker.member_vars["running_accuracy"]*(1.0 + GPA.pc.get_improvement_threshold())}) ' 334 f"which is not lower than " 335 f'{GPA.pai_tracker.member_vars["current_best_validation_score"]}' 336 ) 337 GPA.pai_tracker.member_vars["last_improved_accuracies"].append( 338 GPA.pai_tracker.member_vars["epoch_last_improved"] 339 ) 340 # If it's the first epoch, save as best anyway 341 if len(GPA.pai_tracker.member_vars["accuracies"]) == 1: 342 if GPA.pc.get_verbose(): 343 print("Saving first model or all models") 344 UPA.save_system(net, GPA.pc.get_save_name(), "best_model") 345 if GPA.pc.get_pai_saves(): 346 UPA.pai_save_system(net, GPA.pc.get_save_name(), "best_model")
Check if the new accuracy is a new best.
Performs saves if new best score is found.
Parameters
- net (torch.nn.Module): The neural network model being trained.
- accuracy (float): The accuracy score to check.
- epochs_since_cycle_switch (int): The number of epochs since the last cycle switch.
Returns
- None
def
process_no_improvement(net):
349def process_no_improvement(net): 350 """Handle the case where no improvement is observed. 351 352 If the new dendrite did not improve scores, but its time to switch modes 353 either trigger the end of learning or reset to the previous dendrite 354 to try again. 355 356 Parameters 357 ---------- 358 net : torch.nn.Module 359 The neural network model being trained. 360 361 Returns 362 ------- 363 int 364 The status of restructuring or training completion. 365 torch.nn.Module 366 The potentially modified neural network model. 367 368 """ 369 if GPA.pc.get_verbose(): 370 print( 371 f"Planning to switch to p mode but best beat last: " 372 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]} ' 373 f"current start lr steps: " 374 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 375 f"and last maximum lr steps: " 376 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 377 f'for rate: {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]:.8f}' 378 ) 379 380 now = datetime.now() 381 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 382 383 if GPA.pc.get_verbose(): 384 print( 385 f'1 saving break {dt_string}_noImprove_lr_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 386 ) 387 388 GPA.pai_tracker.save_graphs( 389 f'{dt_string}_noImprove_lr_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 390 ) 391 392 if ( 393 GPA.pai_tracker.member_vars["num_dendrite_tries"] 394 < GPA.pc.get_max_dendrite_tries() -1 395 ): 396 if not GPA.pc.get_silent(): 397 print( 398 f"The newest added dendrites did not improve but current tries " 399 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"] + 1} ' 400 f"is less than max tries {GPA.pc.get_max_dendrite_tries()} " 401 f"so loading last switch and trying new Dendrites." 402 ) 403 old_tries = GPA.pai_tracker.member_vars["num_dendrite_tries"] 404 # Load best model from previous n mode 405 net = UPA.change_learning_modes( 406 net, 407 GPA.pc.get_save_name(), 408 "best_model", 409 GPA.pai_tracker.member_vars["doing_pai"], 410 ) 411 GPA.pai_tracker.member_vars["num_dendrite_tries"] = old_tries + 1 412 return NETWORK_RESTRUCTURED, net 413 else: 414 if not GPA.pc.get_silent(): 415 print( 416 f"The newest added dendrites did not improve system and " 417 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"] + 1} > ' 418 f"{GPA.pc.get_max_dendrite_tries()} so returning training_complete." 419 ) 420 print( 421 "You should now exit your training loop and " 422 "best_model will be your final model for inference" 423 ) 424 if not GPA.pc.get_perforated_backpropagation() and GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 425 print("For improved results, try perforated backpropagation next time!") 426 UPA.load_system(net, GPA.pc.get_save_name(), "best_model", switch_call=True) 427 print('before graphs') 428 GPA.pai_tracker.save_graphs() 429 print('after graphs') 430 UPA.pai_save_system(net, GPA.pc.get_save_name(), "final_clean") 431 print('after save') 432 return TRAINING_COMPLETE, net
Handle the case where no improvement is observed.
If the new dendrite did not improve scores, but its time to switch modes either trigger the end of learning or reset to the previous dendrite to try again.
Parameters
- net (torch.nn.Module): The neural network model being trained.
Returns
- int: The status of restructuring or training completion.
- torch.nn.Module: The potentially modified neural network model.
def
process_final_network(net):
435def process_final_network(net): 436 """When the max number of dendrites has been hit load the best_model and return 437 438 Parameters 439 ---------- 440 net : torch.nn.Module 441 The neural network model being trained. 442 443 Returns 444 ------- 445 torch.nn.Module 446 The final neural network model. 447 """ 448 449 if not GPA.pc.get_silent(): 450 print( 451 f"Last Dendrites were good and this hit the max of {GPA.pc.get_max_dendrites()}" 452 ) 453 if not GPA.pc.get_perforated_backpropagation() and GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 454 print("For improved results, try perforated backpropagation next time!") 455 GPA.pai_tracker.save_graphs("before_final") 456 UPA.load_system(net, GPA.pc.get_save_name(), "best_model", switch_call=True) 457 GPA.pai_tracker.save_graphs() 458 UPA.pai_save_system(net, GPA.pc.get_save_name(), "final_clean") 459 return net
When the max number of dendrites has been hit load the best_model and return
Parameters
- net (torch.nn.Module): The neural network model being trained.
Returns
- torch.nn.Module: The final neural network model.
def
process_scheduler_update(net, accuracy, epochs_since_cycle_switch):
462def process_scheduler_update(net, accuracy, epochs_since_cycle_switch): 463 """Updates the scheduler 464 465 This increments the scheduler, but if we are automatically sweeping 466 to find the best initial learning rate for a new set of dendrites 467 this function also triggers the network at addition time to 468 try the next value. 469 470 Process for finding best initial learning rate for dendrites: 471 1. Start at default rate 472 2. Learn at that rate until scheduler increments twice 473 3. Save that version, start dendrites at LR current increment - 1 474 4. Repeat 2 and 3 until version has worse final score at set LR 475 5. Load previous model with best accuracy at that LR as initial rate 476 477 Parameters 478 ---------- 479 net : torch.nn.Module 480 The neural network model being trained. 481 accuracy : float 482 The accuracy of the model at the current learning rate. 483 epochs_since_cycle_switch : int 484 The number of epochs since the last cycle switch. 485 486 Returns 487 ------- 488 int 489 The status of restructuring or training completion. 490 torch.nn.Module 491 The potentially modified neural network model. 492 """ 493 494 restructured = False 495 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 496 learning_rate1 = param_group["lr"] 497 498 if ( 499 type(GPA.pai_tracker.member_vars["scheduler_instance"]) 500 is torch.optim.lr_scheduler.ReduceLROnPlateau 501 ): 502 if ( 503 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 504 or GPA.pai_tracker.member_vars["mode"] == "p" 505 ): 506 if GPA.pc.get_verbose(): 507 print( 508 f"Updating scheduler with last improved " 509 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]} ' 510 f'from current {GPA.pai_tracker.member_vars["num_epochs_run"]}' 511 ) 512 if GPA.pai_tracker.member_vars["scheduler"] is not None: 513 GPA.pai_tracker.member_vars["scheduler_instance"].step(metrics=accuracy) 514 if ( 515 GPA.pai_tracker.member_vars["scheduler"] 516 is torch.optim.lr_scheduler.ReduceLROnPlateau 517 ): 518 if GPA.pc.get_verbose(): 519 print( 520 f"Scheduler is now at " 521 f'{GPA.pai_tracker.member_vars["scheduler_instance"].num_bad_epochs} bad epochs' 522 ) 523 else: 524 if GPA.pc.get_verbose(): 525 print("Not stepping optimizer since hasnt initialized") 526 527 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 528 if ( 529 epochs_since_cycle_switch > GPA.pc.get_initial_history_after_switches() 530 or GPA.pai_tracker.member_vars["mode"] == "p" 531 ): 532 if GPA.pc.get_verbose(): 533 if hasattr(GPA.pai_tracker.member_vars["scheduler_instance"], '_step_count'): 534 count = GPA.pai_tracker.member_vars["scheduler_instance"]._step_count 535 else: 536 count = GPA.pai_tracker.member_vars["scheduler_instance"].last_epoch 537 538 print( 539 f"Incrementing scheduler to count " 540 f'{count}' 541 ) 542 GPA.pai_tracker.member_vars["scheduler_instance"].step() 543 if ( 544 GPA.pai_tracker.member_vars["scheduler"] 545 is torch.optim.lr_scheduler.ReduceLROnPlateau 546 ): 547 if GPA.pc.get_verbose(): 548 print( 549 f"Scheduler is now at " 550 f'{GPA.pai_tracker.member_vars["scheduler_instance"].num_bad_epochs} bad epochs' 551 ) 552 553 if ( 554 epochs_since_cycle_switch <= GPA.pc.get_initial_history_after_switches() 555 and GPA.pai_tracker.member_vars["mode"] == "n" 556 ): 557 if GPA.pc.get_verbose(): 558 print( 559 f"Not stepping with history {GPA.pc.get_initial_history_after_switches()} " 560 f"and current {epochs_since_cycle_switch}" 561 ) 562 563 for param_group in GPA.pai_tracker.member_vars["optimizer_instance"].param_groups: 564 learning_rate2 = param_group["lr"] 565 566 stepped = False 567 at_last_count = False 568 569 if GPA.pc.get_verbose(): 570 print( 571 f"Checking if at last with scores " 572 f'{len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"])}, ' 573 f"count since switch {epochs_since_cycle_switch} " 574 f"and last total lr step count " 575 f'{GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"]}' 576 ) 577 578 # Check if at double or exactly the test count 579 if ( 580 len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 0 581 and epochs_since_cycle_switch 582 == GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] * 2 583 ) or ( 584 len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 1 585 and epochs_since_cycle_switch 586 == GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] 587 ): 588 at_last_count = True 589 590 if GPA.pc.get_verbose(): 591 print( 592 f"At last count {at_last_count} with count {epochs_since_cycle_switch} " 593 f'and last LR count {GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"]}' 594 ) 595 596 if learning_rate1 != learning_rate2: 597 stepped = True 598 GPA.pai_tracker.member_vars["current_step_count"] += 1 599 600 if GPA.pc.get_verbose(): 601 print( 602 f"Learning rate just stepped to {learning_rate2:.10e} " 603 f'with {GPA.pai_tracker.member_vars["current_step_count"]} total steps' 604 ) 605 606 if ( 607 GPA.pai_tracker.member_vars["current_step_count"] 608 == GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] 609 ): 610 if GPA.pc.get_verbose(): 611 print( 612 f'{GPA.pai_tracker.member_vars["current_step_count"]} ' 613 f"steps is the max of the last switch mode" 614 ) 615 # Set it when 1->2 gets to 2, not when 0->1 hits 2 as stopping point 616 if ( 617 GPA.pai_tracker.member_vars["current_step_count"] 618 - GPA.pai_tracker.member_vars[ 619 "current_n_learning_rate_initial_skip_steps" 620 ] 621 == 1 622 ): 623 GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] = ( 624 epochs_since_cycle_switch 625 ) 626 627 if GPA.pc.get_verbose(): 628 print( 629 f"Learning rates were {learning_rate1:.8e} and {learning_rate2:.8e} " 630 f'started with {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 631 f'and is now at {GPA.pai_tracker.member_vars["current_step_count"]} ' 632 f'committed {GPA.pai_tracker.member_vars["committed_to_initial_rate"]} ' 633 f"then either this (non zero) or eventually comparing to " 634 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 635 f'steps or rate {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]:.8f}' 636 ) 637 638 # If learning rate just stepped, check restart at lower rate 639 if ( 640 (GPA.pai_tracker.member_vars["scheduler"] is not None) 641 and 642 # If potentially might have higher accuracy 643 ( 644 (GPA.pai_tracker.member_vars["mode"] == "n") 645 or GPA.pc.get_learn_dendrites_live() 646 ) 647 and 648 # And learning rate just stepped 649 (stepped or at_last_count) 650 ): 651 652 # If this is the first dendrite addition (last_max_learning_rate_steps == 0), 653 # immediately commit to the initial rate without searching 654 if GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] == 0: 655 if GPA.pc.get_verbose(): 656 print( 657 f"First dendrite addition detected (last_max_learning_rate_steps == 0), " 658 f"immediately committing to initial rate without search" 659 ) 660 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 661 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 662 GPA.pai_tracker.member_vars["current_step_count"] 663 ) 664 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 665 learning_rate2 666 ) 667 668 # If hasn't committed to a learning rate for this cycle yet 669 if not GPA.pai_tracker.member_vars["committed_to_initial_rate"]: 670 best_score_so_far = GPA.pai_tracker.member_vars[ 671 "global_best_validation_score" 672 ] 673 674 if GPA.pc.get_verbose(): 675 print( 676 f"In statements to check next learning rate with " 677 f"stepped {stepped} and max count {at_last_count}" 678 ) 679 680 # If no scores saved for this dendrite and initial LR test did second step 681 if len( 682 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] 683 ) == 0 and ( 684 GPA.pai_tracker.member_vars["current_step_count"] 685 - GPA.pai_tracker.member_vars[ 686 "current_n_learning_rate_initial_skip_steps" 687 ] 688 == 2 689 or at_last_count 690 ): 691 692 restructured = True 693 GPA.pai_tracker.clear_optimizer_and_scheduler() 694 695 # Save system for this initial condition 696 old_global = GPA.pai_tracker.member_vars["global_best_validation_score"] 697 old_accuracy = GPA.pai_tracker.member_vars[ 698 "current_best_validation_score" 699 ] 700 old_counts = GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] 701 skip1 = GPA.pai_tracker.member_vars[ 702 "current_n_learning_rate_initial_skip_steps" 703 ] 704 705 now = datetime.now() 706 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 707 708 GPA.pai_tracker.save_graphs( 709 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 710 ) 711 712 if GPA.pc.get_test_saves(): 713 UPA.save_system( 714 net, 715 GPA.pc.get_save_name(), 716 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 717 ) 718 719 if GPA.pc.get_verbose(): 720 print( 721 f"Saving with initial steps: {dt_string}_PBCount_" 722 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 723 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 724 f"with current best {old_accuracy}" 725 ) 726 727 # Load back at start and try with lower initial learning rate 728 net = UPA.load_system( 729 net, 730 GPA.pc.get_save_name(), 731 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 732 switch_call=True, 733 ) 734 GPA.pai_tracker.member_vars[ 735 "current_n_learning_rate_initial_skip_steps" 736 ] = (skip1 + 1) 737 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"].append( 738 old_accuracy 739 ) 740 GPA.pai_tracker.member_vars["global_best_validation_score"] = old_global 741 GPA.pai_tracker.member_vars["initial_lr_test_epoch_count"] = old_counts 742 743 # If there is one score already, this is first step at next score 744 elif len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 1: 745 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"].append( 746 GPA.pai_tracker.member_vars["current_best_validation_score"] 747 ) 748 749 # If this LR's score was worse than last LR's score 750 lr_score_worse = False 751 if GPA.pai_tracker.member_vars["maximizing_score"]: 752 lr_score_worse = ( 753 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0] 754 > GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1] 755 ) 756 else: 757 lr_score_worse = ( 758 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0] 759 < GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1] 760 ) 761 762 if lr_score_worse: 763 restructured = True 764 GPA.pai_tracker.clear_optimizer_and_scheduler() 765 766 if GPA.pc.get_verbose(): 767 print( 768 f'Got initial {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1} ' 769 f'step score {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0]} ' 770 f'and {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 771 f'score at step {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1]} ' 772 f"so loading old score" 773 ) 774 775 prior_best = GPA.pai_tracker.member_vars[ 776 "current_cycle_lr_max_scores" 777 ][0] 778 779 now = datetime.now() 780 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 781 782 GPA.pai_tracker.save_graphs( 783 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 784 ) 785 786 if GPA.pc.get_test_saves(): 787 UPA.save_system( 788 net, 789 GPA.pc.get_save_name(), 790 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 791 ) 792 793 if GPA.pc.get_verbose(): 794 print( 795 f"Saving with initial steps: {dt_string}_PBCount_" 796 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 797 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 798 ) 799 800 if GPA.pc.get_test_saves(): 801 net = UPA.load_system( 802 net, 803 GPA.pc.get_save_name(), 804 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1}', 805 switch_call=True, 806 ) 807 808 # Save graphs for chosen one 809 now = datetime.now() 810 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 811 812 GPA.pai_tracker.save_graphs( 813 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}PICKED' 814 ) 815 816 if GPA.pc.get_test_saves(): 817 UPA.save_system( 818 net, 819 GPA.pc.get_save_name(), 820 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 821 ) 822 823 if GPA.pc.get_verbose(): 824 print( 825 f"Saving with initial steps: {dt_string}_PBCount_" 826 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 827 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 828 ) 829 830 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 831 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 832 GPA.pai_tracker.member_vars["current_step_count"] 833 ) 834 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 835 learning_rate2 836 ) 837 GPA.pai_tracker.member_vars["current_best_validation_score"] = ( 838 prior_best 839 ) 840 841 if GPA.pc.get_verbose(): 842 print( 843 f"Setting last max steps to " 844 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 845 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 846 ) 847 848 else: # Current LR score is better 849 if GPA.pc.get_verbose(): 850 print( 851 f'Got initial {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]-1} ' 852 f'step score {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][0]} ' 853 f'and {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 854 f'score at step {GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"][1]} ' 855 f"so NOT loading old score and continuing with this score" 856 ) 857 858 if at_last_count: # If this is the last one, set it to be picked 859 restructured = True 860 GPA.pai_tracker.clear_optimizer_and_scheduler() 861 862 now = datetime.now() 863 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 864 865 GPA.pai_tracker.save_graphs( 866 f'{dt_string}_PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}PICKED' 867 ) 868 869 if GPA.pc.get_test_saves(): 870 UPA.save_system( 871 net, 872 GPA.pc.get_save_name(), 873 f'PBCount_{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}', 874 ) 875 876 if GPA.pc.get_verbose(): 877 print( 878 f"Saving with initial steps: {dt_string}_PBCount_" 879 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]}_startSteps_' 880 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}' 881 ) 882 883 GPA.pai_tracker.member_vars["committed_to_initial_rate"] = True 884 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] = ( 885 GPA.pai_tracker.member_vars["current_step_count"] 886 ) 887 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = ( 888 learning_rate2 889 ) 890 891 if GPA.pc.get_verbose(): 892 print( 893 f"Setting last max steps to " 894 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 895 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 896 ) 897 898 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 899 900 elif len(GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"]) == 2: 901 print( 902 "Should never be here. Please let Perforated AI know if this happened." 903 ) 904 pdb.set_trace() 905 906 GPA.pai_tracker.member_vars["global_best_validation_score"] = ( 907 best_score_so_far 908 ) 909 910 else: 911 if GPA.pc.get_verbose(): 912 print( 913 f"Setting last max steps to " 914 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]} ' 915 f'and lr {GPA.pai_tracker.member_vars["last_max_learning_rate_value"]}' 916 ) 917 GPA.pai_tracker.member_vars["last_max_learning_rate_steps"] += 1 918 GPA.pai_tracker.member_vars["last_max_learning_rate_value"] = learning_rate2 919 if restructured: 920 return NETWORK_RESTRUCTURED, net 921 else: 922 return NO_MODEL_UPDATE, net
Updates the scheduler
This increments the scheduler, but if we are automatically sweeping to find the best initial learning rate for a new set of dendrites this function also triggers the network at addition time to try the next value.
Process for finding best initial learning rate for dendrites:
- Start at default rate
- Learn at that rate until scheduler increments twice
- Save that version, start dendrites at LR current increment - 1
- Repeat 2 and 3 until version has worse final score at set LR
- Load previous model with best accuracy at that LR as initial rate
Parameters
- net (torch.nn.Module): The neural network model being trained.
- accuracy (float): The accuracy of the model at the current learning rate.
- epochs_since_cycle_switch (int): The number of epochs since the last cycle switch.
Returns
- int: The status of restructuring or training completion.
- torch.nn.Module: The potentially modified neural network model.
class
PAINeuronModuleTracker:
925class PAINeuronModuleTracker: 926 """ 927 Manager class that tracks all neuron layers and dendrite layers, 928 controls when new dendrites are added, and communicates signals to modules. 929 """ 930 931 def __init__( 932 self, 933 doing_pai, 934 save_name, 935 making_graphs=True, 936 param_vals_setting=-1, 937 values_per_train_epoch=-1, 938 values_per_val_epoch=-1, 939 ): 940 """Initialize the tracker 941 942 Parameters 943 ---------- 944 doing_pai : bool 945 Whether or not dendrites should be used. 946 save_name : str 947 The base name for saving models and graphs. 948 making_graphs : bool, optional 949 Whether or not to generate graphs, by default True. 950 param_vals_setting : int, optional 951 Parameter values setting, by default -1. 952 values_per_train_epoch : int, optional 953 The number of values to look back for graphing 954 during training, by default -1 (all values). 955 values_per_val_epoch : int, optional 956 The number of values to look back for graphing 957 during validation, by default -1 (all values). 958 Returns 959 ------- 960 None 961 """ 962 963 # Dict of member vars and their types for saving 964 self.member_vars = {} 965 self.member_var_types = {} 966 967 # Whether or not PAI will be running 968 self.member_vars["doing_pai"] = doing_pai 969 self.member_var_types["doing_pai"] = "bool" 970 971 # How many Dendrites have been added 972 self.member_vars["num_dendrites_added"] = 0 973 self.member_var_types["num_dendrites_added"] = "int" 974 975 # How many Dendrites have been successfully integrated (kept) 976 self.member_vars["num_dendrites_integrated"] = 0 977 self.member_var_types["num_dendrites_integrated"] = "int" 978 979 # How many cycles have been run, *2 or *2+1 of the above 980 self.member_vars["num_cycles"] = 0 981 self.member_var_types["num_cycles"] = "int" 982 983 # Pointers to all neuron wrapped modules 984 self.neuron_module_vector = [] 985 986 # Pointers to all non neuron modules for tracking 987 self.tracked_neuron_module_vector = [] 988 989 # Neuron training or dendrite training mode 990 self.member_vars["mode"] = "n" 991 self.member_var_types["mode"] = "string" 992 993 # Number of epochs run excluding overwritten epochs 994 self.member_vars["num_epochs_run"] = -1 995 self.member_var_types["num_epochs_run"] = "int" 996 997 # Number including overwritten epochs 998 self.member_vars["total_epochs_run"] = -1 999 self.member_var_types["total_epochs_run"] = "int" 1000 1001 # Last epoch that validation/correlation score was improved 1002 self.member_vars["epoch_last_improved"] = 0 1003 self.member_var_types["epoch_last_improved"] = "int" 1004 1005 # Running validation accuracy 1006 self.member_vars["running_accuracy"] = 0 1007 self.member_var_types["running_accuracy"] = "float" 1008 1009 # True if maxing validation, False if minimizing Loss 1010 self.member_vars["maximizing_score"] = True 1011 self.member_var_types["maximizing_score"] = "bool" 1012 1013 # Mode for switching back and forth between learning modes 1014 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 1015 self.member_var_types["switch_mode"] = "int" 1016 1017 # Epoch of the last switch 1018 self.member_vars["last_switch"] = 0 1019 self.member_var_types["last_switch"] = "int" 1020 1021 # Highest validation score from current cycle 1022 self.member_vars["current_best_validation_score"] = 0 1023 self.member_var_types["current_best_validation_score"] = "float" 1024 1025 # Last epoch where the learning rate was updated 1026 self.member_vars["initial_lr_test_epoch_count"] = -1 1027 self.member_var_types["initial_lr_test_epoch_count"] = "int" 1028 1029 # Highest validation score of full run 1030 self.member_vars["global_best_validation_score"] = 0 1031 self.member_var_types["global_best_validation_score"] = "float" 1032 1033 # List of switch epochs 1034 self.member_vars["switch_epochs"] = [] 1035 self.member_var_types["switch_epochs"] = "int array" 1036 1037 # Parameter counts at each network structure 1038 self.member_vars["param_counts"] = [] 1039 self.member_var_types["param_counts"] = "int array" 1040 1041 # List of epochs where switch was made to neuron training 1042 self.member_vars["n_switch_epochs"] = [] 1043 self.member_var_types["n_switch_epochs"] = "int array" 1044 1045 # List of epochs where switch was made to dendrite training 1046 self.member_vars["p_switch_epochs"] = [] 1047 self.member_var_types["p_switch_epochs"] = "int array" 1048 1049 # List of validation accuracies 1050 self.member_vars["accuracies"] = [] 1051 self.member_var_types["accuracies"] = "float array" 1052 1053 # List of epochs where score improved for scheduler updates 1054 self.member_vars["last_improved_accuracies"] = [] 1055 self.member_var_types["last_improved_accuracies"] = "int array" 1056 1057 # List of test accuracy scores registered 1058 self.member_vars["test_accuracies"] = [] 1059 self.member_var_types["test_accuracies"] = "float array" 1060 1061 # List of accuracies registered during neuron training 1062 self.member_vars["n_accuracies"] = [] 1063 self.member_var_types["n_accuracies"] = "float array" 1064 1065 # List of accuracies registered during dendrite training 1066 self.member_vars["p_accuracies"] = [] 1067 self.member_var_types["p_accuracies"] = "float array" 1068 1069 # Running average accuracies from recent epochs 1070 self.member_vars["running_accuracies"] = [] 1071 self.member_var_types["running_accuracies"] = "float array" 1072 1073 # List of additional scores recorded 1074 self.member_vars["extra_scores"] = {} 1075 self.member_var_types["extra_scores"] = "float array dictionary" 1076 1077 # Extra scores not set to be graphed 1078 self.member_vars["extra_scores_without_graphing"] = {} 1079 self.member_var_types["extra_scores_without_graphing"] = ( 1080 "float array dictionary" 1081 ) 1082 1083 # List of test scores 1084 self.member_vars["test_scores"] = [] 1085 self.member_var_types["test_scores"] = "float array" 1086 1087 # Extra scores calculated during neuron training 1088 self.member_vars["n_extra_scores"] = {} 1089 self.member_var_types["n_extra_scores"] = "float array dictionary" 1090 1091 # List of training losses calculated 1092 self.member_vars["training_loss"] = [] 1093 self.member_var_types["training_loss"] = "float array" 1094 1095 # List of learning rates at each epoch 1096 self.member_vars["training_learning_rates"] = [] 1097 self.member_var_types["training_learning_rates"] = "float array" 1098 1099 # Best dendrite scores 1100 self.member_vars["best_scores"] = [] 1101 self.member_var_types["best_scores"] = "float array array" 1102 1103 # Current dendrite scores 1104 self.member_vars["current_scores"] = [] 1105 self.member_var_types["current_scores"] = "float array array" 1106 1107 # Times for neuron training epochs 1108 self.member_vars["n_epoch_times"] = [] 1109 self.member_var_types["n_epoch_times"] = "float array" 1110 1111 # Timing values 1112 self.member_vars["p_epoch_times"] = [] 1113 self.member_var_types["p_epoch_times"] = "float array" 1114 self.member_vars["n_train_times"] = [] 1115 self.member_var_types["n_train_times"] = "float array" 1116 self.member_vars["p_train_times"] = [] 1117 self.member_var_types["p_train_times"] = "float array" 1118 self.member_vars["n_val_times"] = [] 1119 self.member_var_types["n_val_times"] = "float array" 1120 self.member_vars["p_val_times"] = [] 1121 self.member_var_types["p_val_times"] = "float array" 1122 1123 # Setting for tracking timing 1124 self.member_vars["manual_train_switch"] = False 1125 self.member_var_types["manual_train_switch"] = "bool" 1126 1127 # Tracking scores overwritten when reloading best model 1128 self.member_vars["overwritten_extras"] = [] 1129 self.member_var_types["overwritten_extras"] = "float array dictionary array" 1130 self.member_vars["overwritten_vals"] = [] 1131 self.member_var_types["overwritten_vals"] = "float array array" 1132 self.member_vars["overwritten_epochs"] = 0 1133 self.member_var_types["overwritten_epochs"] = "int" 1134 1135 # Setting for determining scores 1136 self.member_vars["param_vals_setting"] = GPA.pc.get_param_vals_setting() 1137 self.member_var_types["param_vals_setting"] = "int" 1138 1139 # Optimizer and scheduler types and instances 1140 self.member_vars["optimizer"] = None 1141 self.member_var_types["optimizer"] = "type" 1142 self.member_vars["scheduler"] = None 1143 self.member_var_types["scheduler"] = "type" 1144 self.member_vars["optimizer_instance"] = None 1145 self.member_var_types["optimizer_instance"] = "empty array" 1146 self.member_vars["scheduler_instance"] = None 1147 self.member_var_types["scheduler_instance"] = "empty array" 1148 1149 # Flag for if the tracker was loaded 1150 self.loaded = False 1151 1152 # flag for 1153 self.member_vars["step_status"] = STEP_CLEARED 1154 self.member_var_types["step_status"] = "int" 1155 1156 1157 # Settings for tracking learning rates 1158 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 1159 self.member_var_types["current_n_learning_rate_initial_skip_steps"] = "int" 1160 self.member_vars["last_max_learning_rate_steps"] = 0 1161 self.member_var_types["last_max_learning_rate_steps"] = "int" 1162 self.member_vars["last_max_learning_rate_value"] = -1 1163 self.member_var_types["last_max_learning_rate_value"] = "float" 1164 self.member_vars["current_cycle_lr_max_scores"] = [] 1165 self.member_var_types["current_cycle_lr_max_scores"] = "float array" 1166 self.member_vars["current_step_count"] = 0 1167 self.member_var_types["current_step_count"] = "int" 1168 self.member_vars["committed_to_initial_rate"] = True 1169 self.member_var_types["committed_to_initial_rate"] = "bool" 1170 self.member_vars["best_mean_score_improved_this_epoch"] = 0 1171 self.member_var_types["best_mean_score_improved_this_epoch"] = "int" 1172 1173 # Flag for if current dendrite achieved highest global score 1174 self.member_vars["current_n_set_global_best"] = True 1175 self.member_var_types["current_n_set_global_best"] = "bool" 1176 1177 # Number of tries adding this dendrite count 1178 self.member_vars["num_dendrite_tries"] = 0 1179 self.member_var_types["num_dendrite_tries"] = "int" 1180 1181 # Count of batches per epoch 1182 self.values_per_train_epoch = values_per_train_epoch 1183 self.values_per_val_epoch = values_per_val_epoch 1184 1185 self.save_name = save_name 1186 self.making_graphs = making_graphs 1187 1188 self.start_time = time.time() 1189 self.saved_time = 0 1190 self.start_epoch(internal_call=True) 1191 1192 if GPA.pc.get_verbose(): 1193 print(f'Initializing with switch_mode {self.member_vars["switch_mode"]}') 1194 1195 def to_string(self): 1196 """Convert tracker values to string for saving with safetensors.""" 1197 1198 full_string = "" 1199 for var in self.member_vars: 1200 full_string += var + "," 1201 if self.member_vars[var] is None: 1202 full_string += "None" 1203 full_string += "\n" 1204 elif self.member_var_types[var] == "bool": 1205 full_string += str(self.member_vars[var]) 1206 full_string += "\n" 1207 elif self.member_var_types[var] in ("int", "float", "string"): 1208 full_string += str(self.member_vars[var]) 1209 full_string += "\n" 1210 elif self.member_var_types[var] == "type": 1211 name = ( 1212 self.member_vars[var].__module__ 1213 + "." 1214 + self.member_vars[var].__name__ 1215 ) 1216 full_string += str(self.member_vars[var]) 1217 full_string += "\n" 1218 elif self.member_var_types[var] == "empty array": 1219 full_string += "[]" 1220 full_string += "\n" 1221 elif self.member_var_types[var] in ("int array", "float array"): 1222 full_string += "\n" 1223 string = "" 1224 for val in self.member_vars[var]: 1225 string += str(val) + "," 1226 # Remove the last comma 1227 string = string[:-1] 1228 full_string += string 1229 full_string += "\n" 1230 elif self.member_var_types[var] == "float array dictionary array": 1231 full_string += "\n" 1232 for array in self.member_vars[var]: 1233 for key in array: 1234 string = key + "," 1235 for val in array[key]: 1236 string += str(val) + "," 1237 # Remove the last comma 1238 string = string[:-1] 1239 full_string += string 1240 full_string += "\n" 1241 full_string += "endkey" 1242 full_string += "\n" 1243 full_string += "endarray" 1244 full_string += "\n" 1245 elif self.member_var_types[var] == "float array dictionary": 1246 full_string += "\n" 1247 for key in self.member_vars[var]: 1248 string = key + "," 1249 for val in self.member_vars[var][key]: 1250 string += str(val) + "," 1251 # Remove the last comma 1252 string = string[:-1] 1253 full_string += string 1254 full_string += "\n" 1255 full_string += "end" 1256 full_string += "\n" 1257 elif self.member_var_types[var] == "float array array": 1258 full_string += "\n" 1259 for array in self.member_vars[var]: 1260 string = "" 1261 for val in array: 1262 string += str(val) + "," 1263 # Remove the last comma 1264 string = string[:-1] 1265 full_string += string 1266 full_string += "\n" 1267 full_string += "end" 1268 full_string += "\n" 1269 else: 1270 print("Did not find a member variable") 1271 pdb.set_trace() 1272 return full_string 1273 1274 def from_string(self, string): 1275 """Load tracker values from string. 1276 1277 Parameters 1278 ---------- 1279 string : str 1280 The string to load from. 1281 """ 1282 f = io.StringIO(string) 1283 while True: 1284 line = f.readline() 1285 if not line: 1286 break 1287 vals = line.split(",") 1288 var = vals[0] 1289 1290 if self.member_var_types[var] == "bool": 1291 val = vals[1][:-1] 1292 if val == "True": 1293 self.member_vars[var] = True 1294 elif val == "False": 1295 self.member_vars[var] = False 1296 elif val == "1": 1297 self.member_vars[var] = 1 1298 elif val == "0": 1299 self.member_vars[var] = 0 1300 else: 1301 print("Something went wrong with loading") 1302 pdb.set_trace() 1303 elif self.member_var_types[var] == "int": 1304 val = vals[1] 1305 self.member_vars[var] = int(val) 1306 elif self.member_var_types[var] == "float": 1307 val = vals[1] 1308 self.member_vars[var] = float(val) 1309 elif self.member_var_types[var] == "string": 1310 val = vals[1][:-1] 1311 self.member_vars[var] = val 1312 elif self.member_var_types[var] == "type": 1313 # Ignore loading types, tracker should have them set up 1314 continue 1315 elif self.member_var_types[var] == "empty array": 1316 val = vals[1] 1317 self.member_vars[var] = [] 1318 elif self.member_var_types[var] == "int array": 1319 vals = f.readline()[:-1].split(",") 1320 self.member_vars[var] = [] 1321 if vals[0] == "": 1322 continue 1323 for val in vals: 1324 self.member_vars[var].append(int(val)) 1325 elif self.member_var_types[var] == "float array": 1326 vals = f.readline()[:-1].split(",") 1327 self.member_vars[var] = [] 1328 if vals[0] == "": 1329 continue 1330 for val in vals: 1331 self.member_vars[var].append(float(val)) 1332 elif self.member_var_types[var] == "float array dictionary array": 1333 self.member_vars[var] = [] 1334 line2 = f.readline()[:-1] 1335 while line2 != "endarray": 1336 temp = {} 1337 while line2 != "endkey": 1338 vals = line2.split(",") 1339 name = vals[0] 1340 temp[name] = [] 1341 vals = vals[1:] 1342 for val in vals: 1343 temp[name].append(float(val)) 1344 line2 = f.readline()[:-1] 1345 self.member_vars[var].append(temp) 1346 line2 = f.readline()[:-1] 1347 elif self.member_var_types[var] == "float array dictionary": 1348 self.member_vars[var] = {} 1349 line2 = f.readline()[:-1] 1350 while line2 != "end": 1351 vals = line2.split(",") 1352 name = vals[0] 1353 self.member_vars[var][name] = [] 1354 vals = vals[1:] 1355 for val in vals: 1356 self.member_vars[var][name].append(float(val)) 1357 line2 = f.readline()[:-1] 1358 elif self.member_var_types[var] == "float array array": 1359 self.member_vars[var] = [] 1360 line2 = f.readline()[:-1] 1361 while line2 != "end": 1362 vals = line2.split(",") 1363 self.member_vars[var].append([]) 1364 if line2: 1365 for val in vals: 1366 self.member_vars[var][-1].append(float(val)) 1367 line2 = f.readline()[:-1] 1368 else: 1369 print("Did not find a member variable") 1370 1371 pdb.set_trace() 1372 1373 def from_string_debug(self, string): 1374 """Debug function to print tracker values from string without loading them. 1375 1376 Parameters 1377 ---------- 1378 string : str 1379 The string to debug load from. 1380 """ 1381 f = io.StringIO(string) 1382 print("=== DEBUGGING TRACKER VARIABLES ===") 1383 1384 while True: 1385 line = f.readline() 1386 if not line: 1387 break 1388 vals = line.split(",") 1389 var = vals[0] 1390 1391 print(f"\nVariable: {var}") 1392 print(f"Type: {self.member_var_types.get(var, 'UNKNOWN TYPE')}") 1393 print(f"Current value: {self.member_vars.get(var, 'NOT SET')}") 1394 1395 if self.member_var_types.get(var) == "bool": 1396 val = vals[1][:-1] 1397 print(f"Would set to: {val} -> {val == 'True'}") 1398 1399 elif self.member_var_types.get(var) == "int": 1400 val = vals[1] 1401 print(f"Would set to: {int(val)}") 1402 1403 elif self.member_var_types.get(var) == "float": 1404 val = vals[1] 1405 print(f"Would set to: {float(val)}") 1406 1407 elif self.member_var_types.get(var) == "string": 1408 val = vals[1][:-1] 1409 print(f"Would set to: '{val}'") 1410 1411 elif self.member_var_types.get(var) == "type": 1412 print("Would skip (type loading)") 1413 1414 elif self.member_var_types.get(var) == "empty array": 1415 val = vals[1] 1416 print(f"Would set to: [] (empty array)") 1417 1418 elif self.member_var_types.get(var) == "int array": 1419 vals_line = f.readline()[:-1].split(",") 1420 print(f"Would set to int array with {len(vals_line)} elements:") 1421 if vals_line[0] != "": 1422 print( 1423 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1424 ) 1425 else: 1426 print(" Empty array") 1427 1428 elif self.member_var_types.get(var) == "float array": 1429 vals_line = f.readline()[:-1].split(",") 1430 print(f"Would set to float array with {len(vals_line)} elements:") 1431 if vals_line[0] != "": 1432 print( 1433 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1434 ) 1435 else: 1436 print(" Empty array") 1437 1438 elif self.member_var_types.get(var) == "float array dictionary array": 1439 print("Would process float array dictionary array:") 1440 array_count = 0 1441 line2 = f.readline()[:-1] 1442 while line2 != "endarray": 1443 key_count = 0 1444 while line2 != "endkey": 1445 vals_dict = line2.split(",") 1446 name = vals_dict[0] 1447 print( 1448 f" Array {array_count}, Key '{name}': {len(vals_dict)-1} elements" 1449 ) 1450 key_count += 1 1451 line2 = f.readline()[:-1] 1452 print(f" Array {array_count} has {key_count} keys") 1453 array_count += 1 1454 line2 = f.readline()[:-1] 1455 print(f" Total arrays: {array_count}") 1456 1457 elif self.member_var_types.get(var) == "float array dictionary": 1458 print("Would process float array dictionary:") 1459 line2 = f.readline()[:-1] 1460 key_count = 0 1461 while line2 != "end": 1462 vals_dict = line2.split(",") 1463 name = vals_dict[0] 1464 print(f" Key '{name}': {len(vals_dict)-1} elements") 1465 key_count += 1 1466 line2 = f.readline()[:-1] 1467 print(f" Total keys: {key_count}") 1468 1469 elif self.member_var_types.get(var) == "float array array": 1470 print("Would process float array array:") 1471 line2 = f.readline()[:-1] 1472 array_count = 0 1473 while line2 != "end": 1474 if line2: 1475 vals_array = line2.split(",") 1476 print(f" Array {array_count}: {len(vals_array)} elements") 1477 else: 1478 print(f" Array {array_count}: empty") 1479 array_count += 1 1480 line2 = f.readline()[:-1] 1481 print(f" Total arrays: {array_count}") 1482 1483 else: 1484 print(f"UNKNOWN TYPE: {self.member_var_types.get(var, 'NOT FOUND')}") 1485 1486 print("\n=== END DEBUG ===") 1487 1488 def save_tracker_settings(self): 1489 """Save tracker settings for DistributedDataParallel use. 1490 1491 Saves settings in save_name/array_dims.csv 1492 1493 Parameters 1494 ---------- 1495 None 1496 Returns 1497 ------- 1498 None 1499 1500 ----- 1501 Instructions for use are in API customization.md 1502 """ 1503 if not os.path.isdir(self.save_name): 1504 os.makedirs(self.save_name) 1505 f = open(self.save_name + "/array_dims.csv", "w") 1506 for layer in self.neuron_module_vector: 1507 f.write( 1508 f"{layer.name},{layer.dendrite_module.dendrite_values[0].out_channels}\n" 1509 ) 1510 f.close() 1511 if not GPA.pc.get_silent(): 1512 print("Tracker settings saved.") 1513 print("You may now delete save_tracker_settings") 1514 1515 def initialize_tracker_settings(self): 1516 """Initialize tracker settings from saved file. 1517 1518 This function loads tracker settings from a CSV file and applies them 1519 to the layers the tracker is managing. 1520 1521 Parameters 1522 ---------- 1523 None 1524 1525 Returns 1526 ------- 1527 None 1528 1529 """ 1530 1531 channels = {} 1532 if not os.path.exists(self.save_name + "/array_dims.csv"): 1533 print( 1534 "You must call save_tracker_settings before " 1535 "initialize_tracker_settings" 1536 ) 1537 print("Follow instructions in customization.md") 1538 pdb.set_trace() 1539 f = open(self.save_name + "/array_dims.csv", "r") 1540 for line in f: 1541 channels[line.split(",")[0]] = int(line.split(",")[1]) 1542 for layer in self.neuron_module_vector: 1543 layer.dendrite_module.dendrite_values[0].setup_arrays(channels[layer.name]) 1544 1545 def set_optimizer_instance(self, optimizer_instance): 1546 """Set optimizer instance directly. 1547 1548 Parameters 1549 ---------- 1550 optimizer_instance : object 1551 The optimizer instance to set. 1552 1553 Returns 1554 ------- 1555 None 1556 1557 """ 1558 1559 try: 1560 for param_group in optimizer_instance.param_groups: 1561 if ( 1562 param_group["weight_decay"] > 0 1563 and GPA.pc.get_weight_decay_accepted() is False 1564 ): 1565 print( 1566 "For PAI training it is recommended to not use " 1567 "weight decay in your optimizer" 1568 ) 1569 1570 except: 1571 pass 1572 self.member_vars["optimizer_instance"] = optimizer_instance 1573 if GPA.pc.get_perforated_backpropagation(): 1574 TPB.setup_optimizer_pb(self.member_vars["optimizer_instance"]) 1575 1576 def set_optimizer(self, optimizer): 1577 """Set optimizer type to be initialized later 1578 1579 Parameters 1580 ---------- 1581 optimizer : object 1582 The optimizer type to set. 1583 1584 Returns 1585 ------- 1586 None 1587 1588 """ 1589 self.member_vars["optimizer"] = optimizer 1590 1591 def set_scheduler(self, scheduler): 1592 """Set scheduler type to be initialized later 1593 1594 Parameters 1595 ---------- 1596 scheduler : object 1597 The scheduler type to set. 1598 1599 Returns 1600 ------- 1601 None 1602 1603 """ 1604 if scheduler is not torch.optim.lr_scheduler.ReduceLROnPlateau: 1605 if GPA.pc.get_verbose(): 1606 print("Not using ReduceLROnPlateau, this is not recommended") 1607 self.member_vars["scheduler"] = scheduler 1608 1609 def increment_scheduler(self, num_ticks, mode): 1610 """Increment the scheduler a set number of times. 1611 1612 Used for finding best initial learning rate when adding dendrites. 1613 1614 Parameters 1615 ---------- 1616 num_ticks : int 1617 The number of scheduler steps to take. 1618 mode : str 1619 The mode for stepping the scheduler. Options are: 1620 - "step_learning_rate": Step based on improved accuracy epochs 1621 - "increment_epoch_count": Step based on total epoch count 1622 1623 Returns 1624 ------- 1625 current_steps : int 1626 The number of learning rate changes that occurred. 1627 learning_rate1 : float 1628 The final learning rate after stepping. 1629 1630 """ 1631 1632 current_steps = 0 1633 current_ticker = 0 1634 1635 for param_group in GPA.pai_tracker.member_vars[ 1636 "optimizer_instance" 1637 ].param_groups: 1638 learning_rate1 = param_group["lr"] 1639 1640 if GPA.pc.get_verbose(): 1641 print("Using scheduler:") 1642 print(type(self.member_vars["scheduler_instance"])) 1643 1644 while current_ticker < num_ticks: 1645 if GPA.pc.get_verbose(): 1646 print( 1647 f"Lower start rate initial {learning_rate1} " 1648 f'stepping {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} times' 1649 ) 1650 1651 if ( 1652 type(self.member_vars["scheduler_instance"]) 1653 is torch.optim.lr_scheduler.ReduceLROnPlateau 1654 ): 1655 if mode == "step_learning_rate": 1656 # Step with counter as last improved accuracy 1657 self.member_vars["scheduler_instance"].step( 1658 metrics=self.member_vars["last_improved_accuracies"][ 1659 GPA.pai_tracker.steps_after_switch() - 1 1660 ] 1661 ) 1662 elif mode == "increment_epoch_count": 1663 # Step with improved epoch counts up to current location 1664 self.member_vars["scheduler_instance"].step( 1665 metrics=self.member_vars["last_improved_accuracies"][ 1666 -((num_ticks - 1) - current_ticker) - 1 1667 ] 1668 ) 1669 else: 1670 self.member_vars["scheduler_instance"].step() 1671 1672 for param_group in GPA.pai_tracker.member_vars[ 1673 "optimizer_instance" 1674 ].param_groups: 1675 learning_rate2 = param_group["lr"] 1676 1677 if learning_rate2 != learning_rate1: 1678 current_steps += 1 1679 learning_rate1 = learning_rate2 1680 if mode == "step_learning_rate": 1681 current_ticker += 1 1682 if GPA.pc.get_verbose(): 1683 print(f"1 step {current_steps} to {learning_rate2}") 1684 1685 if mode == "increment_epoch_count": 1686 current_ticker += 1 1687 1688 return current_steps, learning_rate1 1689 1690 def setup_optimizer(self, net, opt_args, sched_args=None, parameters=None): 1691 """Initialize the optimizer and scheduler when added. 1692 1693 Parameters 1694 ---------- 1695 net : object 1696 The neural network model. 1697 opt_args : dict 1698 The arguments for the optimizer. 1699 sched_args : dict, optional 1700 The arguments for the scheduler, by default None. 1701 1702 Returns 1703 ------- 1704 optimizer : object 1705 The initialized optimizer instance. 1706 scheduler : object, optional 1707 The initialized scheduler instance, if a scheduler was set. 1708 1709 """ 1710 if "weight_decay" in opt_args and not GPA.pc.get_weight_decay_accepted(): 1711 print( 1712 "For PAI training it is recommended to not use " 1713 "weight decay in your optimizer" 1714 ) 1715 1716 if ("model" not in opt_args.keys()) and "params" not in opt_args.keys(): 1717 print("In setup_optimizer it will be depreciated to not pass in params yourself in the future") 1718 print("please change the settings to include params") 1719 if self.member_vars["mode"] == "n": 1720 if parameters is not None: 1721 opt_args["params"] = parameters 1722 else: 1723 opt_args["params"] = filter(lambda p: p.requires_grad, net.parameters()) 1724 else: 1725 params = UPA.get_pai_network_params(net) 1726 if parameters is not None: 1727 # Filter parameters to only those in params, preserving weight_decay 1728 params_set = set(params) 1729 filtered_params = [] 1730 for param_group in parameters: 1731 filtered_group_params = [p for p in param_group["params"] if p in params_set] 1732 if filtered_group_params: 1733 filtered_params.append({ 1734 "params": filtered_group_params, 1735 "weight_decay": param_group["weight_decay"] 1736 }) 1737 opt_args["params"] = filtered_params 1738 else: 1739 opt_args["params"] = params 1740 elif "params" in opt_args.keys(): 1741 # Check if params is a list of param groups (dicts) or a single param group 1742 params_value = opt_args["params"] 1743 if isinstance(params_value, list) and len(params_value) > 0: 1744 # Check if it's a list of dicts (multiple param groups) or list of tensors (single group) 1745 if isinstance(params_value[0], dict): 1746 # Multiple param groups format: [{"params": [...], "lr": ...}, ...] 1747 # Filter each param group for requires_grad 1748 filtered_param_groups = [] 1749 for param_group in params_value: 1750 filtered_group_params = [p for p in param_group["params"] if p.requires_grad] 1751 if filtered_group_params: 1752 new_group = param_group.copy() 1753 new_group["params"] = filtered_group_params 1754 filtered_param_groups.append(new_group) 1755 opt_args["params"] = filtered_param_groups 1756 else: 1757 # Single param group format: [tensor1, tensor2, ...] or generator 1758 # Filter for requires_grad 1759 opt_args["params"] = [p for p in params_value if p.requires_grad] 1760 elif hasattr(params_value, '__iter__'): 1761 # Handle generators or other iterables 1762 opt_args["params"] = [p for p in params_value if p.requires_grad] 1763 1764 optimizer = self.member_vars["optimizer"](**opt_args) 1765 self.set_optimizer_instance(optimizer) 1766 1767 if self.member_vars["scheduler"] is not None: 1768 # Handle SequentialLR specially 1769 if self.member_vars["scheduler"] is torch.optim.lr_scheduler.SequentialLR: 1770 """ 1771 sched_args should be a dict with "schedulers" (list of tuples) and "milestones" 1772 For example: 1773 sequential_schedArgs = { 1774 "schedulers": [ 1775 (warmup_scheduler_class, warmup_schedArgs), 1776 (main_scheduler_class, main_schedArgs) 1777 ], 1778 "milestones": [switch_epoch] 1779 } 1780 """ 1781 schedulers = [] 1782 milestones = sched_args.get("milestones", []) 1783 scheduler_configs = sched_args.get("schedulers", []) 1784 1785 for scheduler_class, scheduler_args in scheduler_configs: 1786 schedulers.append(scheduler_class(optimizer, **scheduler_args)) 1787 1788 self.member_vars["scheduler_instance"] = torch.optim.lr_scheduler.SequentialLR( 1789 optimizer, schedulers=schedulers, milestones=milestones 1790 ) 1791 else: 1792 self.member_vars["scheduler_instance"] = self.member_vars["scheduler"]( 1793 optimizer, **sched_args 1794 ) 1795 current_steps = 0 1796 1797 for param_group in GPA.pai_tracker.member_vars[ 1798 "optimizer_instance" 1799 ].param_groups: 1800 learning_rate1 = param_group["lr"] 1801 1802 if GPA.pc.get_verbose(): 1803 print( 1804 f"Resetting scheduler with {GPA.pai_tracker.steps_after_switch()} " 1805 f"steps and {GPA.pc.get_initial_history_after_switches()} initial ticks to skip" 1806 ) 1807 1808 # Find setting of previously used learning rate before adding dendrites 1809 if ( 1810 GPA.pai_tracker.member_vars[ 1811 "current_n_learning_rate_initial_skip_steps" 1812 ] 1813 != 0 1814 ): 1815 additional_steps, learning_rate1 = self.increment_scheduler( 1816 GPA.pai_tracker.member_vars[ 1817 "current_n_learning_rate_initial_skip_steps" 1818 ], 1819 "step_learning_rate", 1820 ) 1821 current_steps += additional_steps 1822 1823 if self.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 1824 initial = GPA.pc.get_initial_history_after_switches() 1825 else: 1826 initial = 0 1827 1828 if GPA.pai_tracker.steps_after_switch() > initial: 1829 # Minus extra 1 because this gets called after start epoch 1830 additional_steps, learning_rate1 = self.increment_scheduler( 1831 (GPA.pai_tracker.steps_after_switch() - initial) - 1, 1832 "increment_epoch_count", 1833 ) 1834 current_steps += additional_steps 1835 1836 if GPA.pc.get_verbose(): 1837 print( 1838 f"Scheduler update loop with {current_steps} " 1839 f"ended with {learning_rate1}" 1840 ) 1841 print( 1842 f"Scheduler ended with {current_steps} steps " 1843 f"and lr of {learning_rate1}" 1844 ) 1845 1846 self.member_vars["current_step_count"] = current_steps 1847 return optimizer, self.member_vars["scheduler_instance"] 1848 else: 1849 return optimizer 1850 1851 def clear_optimizer_and_scheduler(self): 1852 """Clear the instances for saving.""" 1853 self.member_vars["optimizer_instance"] = None 1854 self.member_vars["scheduler_instance"] = None 1855 1856 def switch_time(self): 1857 """Determine if it's time to switch between neuron and dendrite training. 1858 1859 Parameters 1860 ---------- 1861 None 1862 1863 Returns 1864 ------- 1865 bool 1866 True if it's time to switch, False otherwise. 1867 1868 Notes 1869 ----- 1870 Based on current settings and history of scores. 1871 """ 1872 1873 switch_phrase = "No mode, this should never be the case." 1874 switch_number = GPA.pc.get_n_epochs_to_switch() 1875 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1876 switch_phrase = "DOING_SWITCH_EVERY_TIME" 1877 elif self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY: 1878 switch_phrase = "DOING_HISTORY" 1879 elif self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH: 1880 switch_phrase = "DOING_FIXED_SWITCH" 1881 switch_number = GPA.pc.get_fixed_switch_num() 1882 elif self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1883 switch_phrase = "DOING_NO_SWITCH" 1884 else: 1885 print( 1886 "A switch mode must be set. Check your settings for GPA.pc.set_switch_mode()." 1887 ) 1888 pdb.set_trace() 1889 if not GPA.pc.get_silent(): 1890 if(GPA.pc.get_perforated_backpropagation()): 1891 print( 1892 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1893 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1894 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1895 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1896 f'n: {switch_number}, p: {GPA.pc.get_p_epochs_to_switch()}, ' 1897 f'num_cycles: {self.member_vars["num_cycles"]}' 1898 ) 1899 else: 1900 print( 1901 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1902 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1903 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1904 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1905 f'n: {switch_number}, num_cycles: {self.member_vars["num_cycles"]}' 1906 ) 1907 print( 1908 f' Score tracking: current_n_set_global_best={self.member_vars["current_n_set_global_best"]}, ' 1909 f'global_best={self.member_vars["global_best_validation_score"]:.4f}, ' 1910 f'current_best={self.member_vars["current_best_validation_score"]:.4f}' 1911 ) 1912 if GPA.pc.get_perforated_backpropagation(): 1913 # this will fill in epoch last improved 1914 TPB.best_pai_score_improved_this_epoch(self) ## CLOSED ONLY 1915 if self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1916 if not GPA.pc.get_silent(): 1917 print("Returning False - doing no switch mode") 1918 return False 1919 1920 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1921 if not GPA.pc.get_silent(): 1922 print("Returning True - switching every time") 1923 return True 1924 1925 # Check if we're in the middle of learning rate optimization 1926 # If so, block ALL switch triggers until committed 1927 if GPA.pc.get_verbose(): 1928 print("=== LR Optimization Check ===") 1929 print(f' mode == "n": {self.member_vars["mode"] == "n"}') 1930 print(f" get_learn_dendrites_live(): {GPA.pc.get_learn_dendrites_live()}") 1931 print(f' committed_to_initial_rate: {GPA.pai_tracker.member_vars["committed_to_initial_rate"]}') 1932 print(f" get_dont_give_up_unless_learning_rate_lowered(): {GPA.pc.get_dont_give_up_unless_learning_rate_lowered()}") 1933 print(f' current_n_learning_rate_initial_skip_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"]}') 1934 print(f' last_max_learning_rate_steps: {self.member_vars["last_max_learning_rate_steps"]}') 1935 print(f' skip_steps < max_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"] < self.member_vars["last_max_learning_rate_steps"]}') 1936 print(f' scheduler is not None: {self.member_vars["scheduler"] is not None}') 1937 print("=============================") 1938 1939 if ( 1940 ((self.member_vars["mode"] == "n") or GPA.pc.get_learn_dendrites_live()) 1941 and (GPA.pai_tracker.member_vars["committed_to_initial_rate"] is False) 1942 and (GPA.pc.get_dont_give_up_unless_learning_rate_lowered()) 1943 and ( 1944 self.member_vars["current_n_learning_rate_initial_skip_steps"] 1945 <= self.member_vars["last_max_learning_rate_steps"] 1946 ) 1947 and self.member_vars["scheduler"] is not None 1948 ): 1949 if not GPA.pc.get_silent(): 1950 print( 1951 f"Returning False - learning rate optimization in progress. " 1952 f"Not committed yet. Comparing " 1953 f'initial {self.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 1954 f'to last max {self.member_vars["last_max_learning_rate_steps"]}' 1955 ) 1956 return False 1957 1958 if len(self.member_vars["switch_epochs"]) == 0: 1959 this_count = self.member_vars["num_epochs_run"] 1960 else: 1961 this_count = ( 1962 self.member_vars["num_epochs_run"] 1963 - self.member_vars["switch_epochs"][-1] 1964 ) 1965 cap_switch = False 1966 if GPA.pc.get_perforated_backpropagation(): 1967 cap_switch = TPB.check_cap_switch(self, this_count) 1968 1969 if self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY and ( 1970 ( 1971 (self.member_vars["mode"] == "n") 1972 and ( 1973 self.member_vars["num_epochs_run"] 1974 - self.member_vars["epoch_last_improved"] 1975 >= GPA.pc.get_n_epochs_to_switch() 1976 ) 1977 and this_count 1978 >= GPA.pc.get_initial_history_after_switches() 1979 + GPA.pc.get_n_epochs_to_switch() 1980 ) 1981 or (GPA.pc.get_perforated_backpropagation() and TPB.history_switch(self)) 1982 or cap_switch 1983 ): 1984 if not GPA.pc.get_silent(): 1985 print("Returning True - History and last improved is hit") 1986 return True 1987 1988 if self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH and ( 1989 ( 1990 self.member_vars["total_epochs_run"] % GPA.pc.get_fixed_switch_num() 1991 == GPA.pc.get_fixed_switch_num() - 1 1992 ) 1993 and self.member_vars["num_epochs_run"] 1994 >= GPA.pc.get_first_fixed_switch_num() - 1 1995 ): 1996 if not GPA.pc.get_silent(): 1997 print("Returning True - Fixed switch number is hit") 1998 return True 1999 2000 if not GPA.pc.get_silent(): 2001 print("Returning False - no triggers to switch have been hit") 2002 return False 2003 2004 def steps_after_switch(self): 2005 """Based on settings, return value for steps since a switch. 2006 2007 Different options for param vals setting determine what is returned. 2008 2009 Parameters 2010 ---------- 2011 None 2012 2013 Returns 2014 ------- 2015 int 2016 The number of epochs since the last switch, or total epochs run, 2017 depending on settings. 2018 2019 """ 2020 if self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_TOTAL_EPOCH: 2021 return self.member_vars["num_epochs_run"] 2022 elif ( 2023 self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_UPDATE_EPOCH 2024 ): 2025 return self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2026 elif ( 2027 self.member_vars["param_vals_setting"] 2028 == GPA.pc.PARAM_VALS_BY_NEURON_EPOCH_START 2029 ): 2030 if self.member_vars["mode"] == "p": 2031 return ( 2032 self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2033 ) 2034 else: 2035 return self.member_vars["num_epochs_run"] 2036 else: 2037 print( 2038 f'{self.member_vars["param_vals_setting"]} is not a valid param vals option' 2039 ) 2040 pdb.set_trace() 2041 2042 def add_pai_neuron_module(self, new_module, initial_add=True): 2043 """Add neuron modules to internal vectors. 2044 2045 Parameters 2046 ---------- 2047 new_module : object 2048 The new module to add. 2049 initial_add : bool, optional 2050 Whether this is the initial addition rather than loading from file 2051 2052 Returns 2053 ------- 2054 None 2055 2056 """ 2057 2058 # If it's a duplicate, ignore the second addition 2059 if new_module in self.neuron_module_vector: 2060 return 2061 self.neuron_module_vector.append(new_module) 2062 if self.member_vars["doing_pai"]: 2063 PA.set_wrapped_params(new_module) 2064 if initial_add: 2065 self.member_vars["best_scores"].append([]) 2066 self.member_vars["current_scores"].append([]) 2067 2068 def add_tracked_neuron_module(self, new_module, initial_add=True): 2069 """Add tracked modules to internal vectors 2070 2071 Parameters 2072 ---------- 2073 new_module : object 2074 The new module to add. 2075 initial_add : bool, optional 2076 Whether this is the initial addition rather than loading from file 2077 2078 Returns 2079 ------- 2080 None 2081 2082 """ 2083 # If it's a duplicate, ignore the second addition 2084 if new_module in self.tracked_neuron_module_vector: 2085 return 2086 self.tracked_neuron_module_vector.append(new_module) 2087 if self.member_vars["doing_pai"]: 2088 PA.set_tracked_params(new_module) 2089 2090 def reset_module_vector(self, net, load_from_restart): 2091 """Clear internal vectors and reset from network. 2092 2093 Parameters 2094 ---------- 2095 net : object 2096 The neural network model. 2097 load_from_restart : bool 2098 Whether loading from a restart file. 2099 2100 Returns 2101 ------- 2102 None 2103 2104 """ 2105 self.neuron_module_vector = [] 2106 self.tracked_neuron_module_vector = [] 2107 this_list = UPA.get_pai_modules(net, 0) 2108 for module in this_list: 2109 self.add_pai_neuron_module(module, initial_add=load_from_restart) 2110 this_list = UPA.get_tracked_modules(net, 0) 2111 for module in this_list: 2112 self.add_tracked_neuron_module(module, initial_add=load_from_restart) 2113 2114 def reset_vals_for_score_reset(self): 2115 """Reset cycle scores for new cycle.""" 2116 2117 if GPA.pc.get_find_best_lr(): 2118 self.member_vars["committed_to_initial_rate"] = False 2119 print("Resetting committed to initial rate to False") 2120 # If retaining all dendrties always say that the current dendrites set global best for saving and loading 2121 if GPA.pc.get_retain_all_dendrites(): 2122 self.member_vars["current_n_set_global_best"] = True 2123 self.member_vars["global_best_validation_score"] = 0 2124 else: 2125 self.member_vars["current_n_set_global_best"] = False 2126 2127 # Don't reset global best, but do reset current best 2128 self.member_vars["current_best_validation_score"] = 0 2129 self.member_vars["initial_lr_test_epoch_count"] = -1 2130 2131 def set_dendrite_training(self): 2132 """Signal all layers to start dendrite training.""" 2133 if GPA.pc.get_verbose(): 2134 print("Calling set_dendrite_training") 2135 2136 for layer in self.neuron_module_vector[:]: 2137 worked = layer.set_mode("p") 2138 """ 2139 worked is False when a layer was added to the neuron module vector 2140 but then it's never actually been used. This can happen when 2141 you have set a layer to have requires_grad = False or when 2142 you have a module as a member variable but it's not actually 2143 part of the network. Should be moved to be a tracked layer 2144 rather than a neuron layer. 2145 """ 2146 if not worked: 2147 self.neuron_module_vector.remove(layer) 2148 2149 for layer in self.tracked_neuron_module_vector[:]: 2150 worked = layer.set_mode("p") 2151 2152 self.create_new_dendrite_module() 2153 self.member_vars["mode"] = "p" 2154 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2155 2156 if GPA.pc.get_learn_dendrites_live(): 2157 self.reset_vals_for_score_reset() 2158 2159 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2160 "current_step_count" 2161 ] 2162 2163 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 2164 GPA.pai_tracker.member_vars["num_cycles"] += 1 2165 2166 2167 def set_neuron_training(self): 2168 """Signal all layers to start neuron training.""" 2169 for module in self.neuron_module_vector: 2170 module.set_mode("n") 2171 for module in self.tracked_neuron_module_vector[:]: 2172 module.set_mode("n") 2173 2174 self.member_vars["mode"] = "n" 2175 self.member_vars["num_dendrites_added"] += 1 2176 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2177 self.reset_vals_for_score_reset() 2178 2179 self.member_vars["current_cycle_lr_max_scores"] = [] 2180 if GPA.pc.get_learn_dendrites_live(): 2181 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2182 "current_step_count" 2183 ] 2184 GPA.pai_tracker.member_vars["num_cycles"] += 1 2185 2186 if GPA.pc.get_reset_best_score_on_switch(): 2187 GPA.pai_tracker.member_vars["current_best_validation_score"] = 0 2188 GPA.pai_tracker.member_vars["running_accuracy"] = 0 2189 2190 def start_epoch(self, internal_call=False): 2191 """Perform steps for when a new training epoch is about to begin. 2192 2193 Parameters 2194 ---------- 2195 internal_call : bool, optional 2196 Whether this is an internal call or manual call 2197 2198 Returns 2199 ------- 2200 None 2201 2202 Notes 2203 ----- 2204 If you ever need to call this manually, set internal_call to False. 2205 2206 """ 2207 if self.member_vars["manual_train_switch"] and internal_call: 2208 return 2209 2210 if not internal_call and not self.member_vars["manual_train_switch"]: 2211 self.member_vars["manual_train_switch"] = True 2212 self.saved_time = 0 2213 self.member_vars["num_epochs_run"] = -1 2214 self.member_vars["total_epochs_run"] = -1 2215 2216 end = time.time() 2217 if self.member_vars["manual_train_switch"]: 2218 if self.saved_time != 0: 2219 if self.member_vars["mode"] == "p": 2220 self.member_vars["p_val_times"].append(end - self.saved_time) 2221 else: 2222 self.member_vars["n_val_times"].append(end - self.saved_time) 2223 2224 if self.member_vars["mode"] == "p": 2225 for layer in self.neuron_module_vector: 2226 for m in range(0, GPA.pc.get_global_candidates()): 2227 with torch.no_grad(): 2228 if GPA.pc.get_verbose(): 2229 print(f"Resetting score for {layer.name}") 2230 # Snapshot best_score before reset so we can compute per-epoch improvement 2231 layer.dendrite_module.dendrite_values[ 2232 m 2233 ].epoch_start_best_score.copy_( 2234 layer.dendrite_module.dendrite_values[ 2235 m 2236 ].best_score.detach() 2237 ) 2238 layer.dendrite_module.dendrite_values[ 2239 m 2240 ].best_score_improved_this_epoch = ( 2241 layer.dendrite_module.dendrite_values[ 2242 m 2243 ].best_score_improved_this_epoch 2244 * 0 2245 ) 2246 layer.dendrite_module.dendrite_values[ 2247 m 2248 ].nodes_best_improved_this_epoch = ( 2249 layer.dendrite_module.dendrite_values[ 2250 m 2251 ].nodes_best_improved_this_epoch 2252 * 0 2253 ) 2254 layer.dendrite_module.dendrite_values[ 2255 m 2256 ].nodes_improved_any = ( 2257 layer.dendrite_module.dendrite_values[ 2258 m 2259 ].nodes_improved_any 2260 * 0 2261 ) 2262 if GPA.pc.get_perforated_backpropagation(): 2263 self.member_vars["best_mean_score_improved_this_epoch"] = 0 2264 self.member_vars["num_epochs_run"] += 1 2265 self.member_vars["total_epochs_run"] = ( 2266 self.member_vars["num_epochs_run"] + self.member_vars["overwritten_epochs"] 2267 ) 2268 self.saved_time = end 2269 2270 def stop_epoch(self, internal_call=False): 2271 """Perform steps when a training epoch has completed. 2272 2273 Parameters 2274 ---------- 2275 internal_call : bool, optional 2276 Whether this is an internal call or manual call 2277 2278 Returns 2279 ------- 2280 None 2281 2282 Notes 2283 ----- 2284 If you ever need to call this manually, set internal_call to False. 2285 2286 """ 2287 end = time.time() 2288 if self.member_vars["manual_train_switch"] and internal_call: 2289 return 2290 2291 if self.member_vars["manual_train_switch"]: 2292 if self.member_vars["mode"] == "p": 2293 self.member_vars["p_train_times"].append(end - self.saved_time) 2294 else: 2295 self.member_vars["n_train_times"].append(end - self.saved_time) 2296 else: 2297 if self.member_vars["mode"] == "p": 2298 self.member_vars["p_epoch_times"].append(end - self.saved_time) 2299 else: 2300 self.member_vars["n_epoch_times"].append(end - self.saved_time) 2301 2302 self.saved_time = end 2303 2304 def initialize( 2305 self, 2306 model, 2307 doing_pai=True, 2308 save_name="PAI", 2309 making_graphs=True, 2310 maximizing_score=True, 2311 num_classes=10000, 2312 values_per_train_epoch=-1, 2313 values_per_val_epoch=-1, 2314 zooming_graph=True, 2315 ): 2316 """Setup the tracker with initial settings. 2317 2318 2319 Parameters 2320 ---------- 2321 model : object 2322 The neural network model. 2323 doing_pai : bool, optional 2324 Whether to add dendrites, by default True. 2325 save_name : str, optional 2326 The name under which to save the model. 2327 making_graphs : bool, optional 2328 Whether to make graphs, by default True. 2329 maximizing_score : bool, optional 2330 Whether to maximize the score, by default True. 2331 num_classes : int, optional 2332 The number of classes in the dataset, unused 2333 values_per_train_epoch : int, optional 2334 The number of values to look back for graphing 2335 during training, by default -1 (all values). 2336 values_per_val_epoch : int, optional 2337 The number of values to look back for graphing 2338 during validation, by default -1 (all values). 2339 zooming_graph : bool, optional 2340 Whether to zoom on graphs, by default True. 2341 2342 """ 2343 model = UPA.convert_network(model) 2344 self.member_vars["doing_pai"] = doing_pai 2345 self.member_vars["maximizing_score"] = maximizing_score 2346 self.save_name = save_name 2347 self.zooming_graph = zooming_graph 2348 self.making_graphs = making_graphs 2349 2350 if not self.loaded: 2351 self.member_vars["running_accuracy"] = (1.0 / num_classes) * 100 2352 2353 self.values_per_train_epoch = values_per_train_epoch 2354 self.values_per_val_epoch = values_per_val_epoch 2355 2356 if GPA.pc.get_testing_dendrite_capacity(): 2357 if not GPA.pc.get_silent(): 2358 print("Running a test of Dendrite Capacity.") 2359 GPA.pc.set_switch_mode(GPA.pc.DOING_SWITCH_EVERY_TIME) 2360 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 2361 GPA.pc.set_retain_all_dendrites(True) 2362 GPA.pc.set_max_dendrite_tries(1000) 2363 GPA.pc.set_max_dendrites(1000) 2364 if GPA.pc.get_perforated_backpropagation(): 2365 GPA.pc.set_initial_correlation_batches(1) 2366 else: 2367 if not GPA.pc.get_silent(): 2368 print("Running Dendrite Experiment") 2369 return model 2370 2371 def generate_accuracy_plots(self, ax, save_folder, extra_string): 2372 """ 2373 Generate plots and csvs for accuracy 2374 2375 Parameters 2376 ---------- 2377 ax : object 2378 The matplotlib axis to plot on. 2379 save_folder : str 2380 The folder to save the plots and csvs in. 2381 extra_string : str 2382 An extra string to append to the filenames. 2383 2384 Returns 2385 ------- 2386 None 2387 2388 """ 2389 2390 # If scores are being saved for epochs that get overwritten, plot them 2391 for list_id in range(len(self.member_vars["overwritten_extras"])): 2392 for extra_id in self.member_vars["overwritten_extras"][list_id]: 2393 ax.plot( 2394 np.arange( 2395 len(self.member_vars["overwritten_extras"][list_id][extra_id]) 2396 ), 2397 self.member_vars["overwritten_extras"][list_id][extra_id], 2398 "r", 2399 ) 2400 ax.plot( 2401 np.arange(len(self.member_vars["overwritten_vals"][list_id])), 2402 self.member_vars["overwritten_vals"][list_id], 2403 "b", 2404 ) 2405 2406 # Determine which accuracy vector to use 2407 if GPA.pc.get_drawing_pai(): 2408 accuracies = self.member_vars["accuracies"] 2409 else: 2410 accuracies = self.member_vars["n_accuracies"] 2411 2412 # Get pointer to additional scores being saved 2413 extra_scores = self.member_vars["extra_scores"] 2414 2415 # Plot the main accuracy scores 2416 ax.plot(np.arange(len(accuracies)), accuracies, label="Validation Scores") 2417 ax.plot( 2418 np.arange(len(self.member_vars["running_accuracies"])), 2419 self.member_vars["running_accuracies"], 2420 label="Validation Running Scores", 2421 ) 2422 2423 # Plot additional scores 2424 for extra_score in extra_scores: 2425 ax.plot( 2426 np.arange(len(extra_scores[extra_score])), 2427 extra_scores[extra_score], 2428 label=extra_score, 2429 ) 2430 2431 plt.title(save_folder + "/" + self.save_name + "Scores") 2432 plt.xlabel("Epochs") 2433 plt.ylabel("Score") 2434 2435 # Add point at epoch last improved and best validation score 2436 if GPA.pc.get_drawing_pai(): 2437 ax.plot( 2438 self.member_vars["epoch_last_improved"], 2439 self.member_vars["global_best_validation_score"], 2440 "bo", 2441 label="Global best (y)", 2442 ) 2443 ax.plot( 2444 self.member_vars["epoch_last_improved"], 2445 accuracies[self.member_vars["epoch_last_improved"]], 2446 "go", 2447 label="Epoch Last Improved", 2448 ) 2449 else: 2450 if self.member_vars["mode"] == "n": 2451 missed_time = ( 2452 self.member_vars["num_epochs_run"] 2453 - self.member_vars["epoch_last_improved"] 2454 ) 2455 ax.plot( 2456 (len(self.member_vars["n_accuracies"]) - 1) - missed_time, 2457 self.member_vars["n_accuracies"][-(missed_time + 1)], 2458 "go", 2459 label="Epoch Last Improved", 2460 ) 2461 2462 # Generate csv file for the values graphed 2463 pd1 = pd.DataFrame( 2464 {"Epochs": np.arange(len(accuracies)), "Validation Scores": accuracies} 2465 ) 2466 pd2 = pd.DataFrame( 2467 { 2468 "Epochs": np.arange(len(self.member_vars["running_accuracies"])), 2469 "Validation Running Scores": self.member_vars["running_accuracies"], 2470 } 2471 ) 2472 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2473 for extra_score in extra_scores: 2474 pd2 = pd.DataFrame( 2475 { 2476 "Epochs": np.arange(len(extra_scores[extra_score])), 2477 extra_score: extra_scores[extra_score], 2478 } 2479 ) 2480 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2481 extra_scores_without_graphing = self.member_vars[ 2482 "extra_scores_without_graphing" 2483 ] 2484 for extra_score in extra_scores_without_graphing: 2485 pd2 = pd.DataFrame( 2486 { 2487 "Epochs": np.arange( 2488 len(extra_scores_without_graphing[extra_score]) 2489 ), 2490 extra_score: extra_scores_without_graphing[extra_score], 2491 } 2492 ) 2493 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2494 pd1.to_csv( 2495 save_folder + "/" + self.save_name + extra_string + "Scores.csv", 2496 index=False, 2497 ) 2498 del pd1, pd2 2499 2500 # Set y min and max to zoom in on important part of axis 2501 if ( 2502 len(self.member_vars["switch_epochs"]) > 0 2503 and self.member_vars["switch_epochs"][0] > 0 2504 and self.zooming_graph 2505 ): 2506 if GPA.pai_tracker.member_vars["maximizing_score"]: 2507 min_val = np.array( 2508 accuracies[0 : self.member_vars["switch_epochs"][0]] 2509 ).mean() 2510 for extra_score in extra_scores: 2511 min_pot = np.array( 2512 extra_scores[extra_score][ 2513 0 : self.member_vars["switch_epochs"][0] 2514 ] 2515 ).mean() 2516 if min_pot < min_val: 2517 min_val = min_pot 2518 ax.set_ylim(ymin=min_val) 2519 else: 2520 max_val = np.array( 2521 accuracies[0 : self.member_vars["switch_epochs"][0]] 2522 ).mean() 2523 for extra_score in extra_scores: 2524 max_pot = np.array( 2525 extra_scores[extra_score][ 2526 0 : self.member_vars["switch_epochs"][0] 2527 ] 2528 ).mean() 2529 if max_pot > max_val: 2530 max_val = max_pot 2531 ax.set_ylim(ymax=max_val) 2532 2533 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2534 2535 # Draw vertical lines for epochs where a dendrite switch occurred 2536 if GPA.pc.get_drawing_pai() and self.member_vars["doing_pai"]: 2537 color = "r" 2538 for switcher in self.member_vars["switch_epochs"]: 2539 plt.axvline(x=switcher, ymin=0, ymax=1, color=color) 2540 if color == "r": 2541 color = "b" 2542 else: 2543 color = "r" 2544 else: 2545 for switcher in self.member_vars["n_switch_epochs"]: 2546 plt.axvline(x=switcher, ymin=0, ymax=1, color="b") 2547 2548 def generate_time_plots(self, ax, save_folder, extra_string): 2549 """ 2550 Generate plots and csvs for timing 2551 2552 Parameters 2553 ---------- 2554 ax : object 2555 The matplotlib axis to plot on. 2556 save_folder : str 2557 The folder to save the plots and csvs in. 2558 extra_string : str 2559 An extra string to append to the filenames. 2560 2561 Returns 2562 ------- 2563 None 2564 2565 """ 2566 if self.member_vars["manual_train_switch"]: 2567 ax.plot( 2568 np.arange(len(self.member_vars["n_train_times"])), 2569 self.member_vars["n_train_times"], 2570 label="Normal Epoch Train Times", 2571 ) 2572 ax.plot( 2573 np.arange(len(self.member_vars["p_train_times"])), 2574 self.member_vars["p_train_times"], 2575 label="PAI Epoch Train Times", 2576 ) 2577 ax.plot( 2578 np.arange(len(self.member_vars["n_val_times"])), 2579 self.member_vars["n_val_times"], 2580 label="Normal Epoch Val Times", 2581 ) 2582 ax.plot( 2583 np.arange(len(self.member_vars["p_val_times"])), 2584 self.member_vars["p_val_times"], 2585 label="PAI Epoch Val Times", 2586 ) 2587 2588 plt.title( 2589 save_folder + "/" + self.save_name + "times (by train() and eval())" 2590 ) 2591 plt.xlabel("Iteration") 2592 plt.ylabel("Epoch Time in Seconds ") 2593 ax.set_ylim(ymin=0) 2594 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2595 2596 pd1 = pd.DataFrame( 2597 { 2598 "Epochs": np.arange(len(self.member_vars["n_train_times"])), 2599 "Normal Epoch Train Times": self.member_vars["n_train_times"], 2600 } 2601 ) 2602 pd2 = pd.DataFrame( 2603 { 2604 "Epochs": np.arange(len(self.member_vars["p_train_times"])), 2605 "PAI Epoch Train Times": self.member_vars["p_train_times"], 2606 } 2607 ) 2608 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2609 2610 pd2 = pd.DataFrame( 2611 { 2612 "Epochs": np.arange(len(self.member_vars["n_val_times"])), 2613 "Normal Epoch Val Times": self.member_vars["n_val_times"], 2614 } 2615 ) 2616 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2617 2618 pd2 = pd.DataFrame( 2619 { 2620 "Epochs": np.arange(len(self.member_vars["p_val_times"])), 2621 "PAI Epoch Val Times": self.member_vars["p_val_times"], 2622 } 2623 ) 2624 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2625 2626 pd1.to_csv( 2627 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2628 index=False, 2629 ) 2630 del pd1, pd2 2631 else: 2632 ax.plot( 2633 np.arange(len(self.member_vars["n_epoch_times"])), 2634 self.member_vars["n_epoch_times"], 2635 label="Normal Epoch Times", 2636 ) 2637 ax.plot( 2638 np.arange(len(self.member_vars["p_epoch_times"])), 2639 self.member_vars["p_epoch_times"], 2640 label="PAI Epoch Times", 2641 ) 2642 2643 plt.title( 2644 save_folder + "/" + self.save_name + "times (by train() and eval())" 2645 ) 2646 plt.xlabel("Iteration") 2647 plt.ylabel("Epoch Time in Seconds ") 2648 ax.set_ylim(ymin=0) 2649 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2650 2651 pd1 = pd.DataFrame( 2652 { 2653 "Epochs": np.arange(len(self.member_vars["n_epoch_times"])), 2654 "Normal Epoch Times": self.member_vars["n_epoch_times"], 2655 } 2656 ) 2657 pd2 = pd.DataFrame( 2658 { 2659 "Epochs": np.arange(len(self.member_vars["p_epoch_times"])), 2660 "PAI Epoch Times": self.member_vars["p_epoch_times"], 2661 } 2662 ) 2663 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2664 2665 pd1.to_csv( 2666 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2667 index=False, 2668 ) 2669 del pd1, pd2 2670 2671 if self.values_per_train_epoch != -1 and self.values_per_val_epoch != -1: 2672 ax2 = ax.twinx() # Second axes sharing same x-axis 2673 ax2.set_ylabel("Single Datapoint Time in Seconds") 2674 2675 ax2.plot( 2676 np.arange(len(self.member_vars["n_train_times"])), 2677 np.array(self.member_vars["n_train_times"]) 2678 / self.values_per_train_epoch, 2679 linestyle="dashed", 2680 label="Normal Train Item Times", 2681 ) 2682 ax2.plot( 2683 np.arange(len(self.member_vars["p_train_times"])), 2684 np.array(self.member_vars["p_train_times"]) 2685 / self.values_per_train_epoch, 2686 linestyle="dashed", 2687 label="PAI Train Item Times", 2688 ) 2689 ax2.plot( 2690 np.arange(len(self.member_vars["n_val_times"])), 2691 np.array(self.member_vars["n_val_times"]) / self.values_per_val_epoch, 2692 linestyle="dashed", 2693 label="Normal Val Item Times", 2694 ) 2695 ax2.plot( 2696 np.arange(len(self.member_vars["p_val_times"])), 2697 np.array(self.member_vars["p_val_times"]) / self.values_per_val_epoch, 2698 linestyle="dashed", 2699 label="PAI Val Item Times", 2700 ) 2701 ax2.tick_params(axis="y") 2702 ax2.set_ylim(ymin=0) 2703 ax2.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2704 2705 def generate_learning_rate_plots(self, ax, save_folder, extra_string): 2706 """ 2707 Generate plots and csvs for learning rate 2708 2709 Parameters 2710 ---------- 2711 ax : object 2712 The matplotlib axis to plot on. 2713 save_folder : str 2714 The folder to save the plots and csvs in. 2715 extra_string : str 2716 An extra string to append to the filenames. 2717 2718 Returns 2719 ------- 2720 None 2721 2722 """ 2723 ax.plot( 2724 np.arange(len(self.member_vars["training_learning_rates"])), 2725 self.member_vars["training_learning_rates"], 2726 label="learning_rate", 2727 ) 2728 plt.title(save_folder + "/" + self.save_name + "learning_rate") 2729 plt.xlabel("Epochs") 2730 plt.ylabel("learning_rate") 2731 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2732 2733 pd1 = pd.DataFrame( 2734 { 2735 "Epochs": np.arange(len(self.member_vars["training_learning_rates"])), 2736 "learning_rate": self.member_vars["training_learning_rates"], 2737 } 2738 ) 2739 pd1.to_csv( 2740 save_folder + "/" + self.save_name + extra_string + "learning_rate.csv", 2741 index=False, 2742 ) 2743 del pd1 2744 2745 def generate_dendrite_learning_plots(self, ax, save_folder, extra_string): 2746 """ 2747 Generate dendrite score plots for the tracker. 2748 Also saves csv files associated with the plots. 2749 """ 2750 if self.member_vars["doing_pai"]: 2751 pd1 = None 2752 pd2 = None 2753 num_colors = len(self.neuron_module_vector) 2754 2755 if ( 2756 len(self.neuron_module_vector) > 0 2757 and len(self.member_vars["current_scores"][0]) != 0 2758 ): 2759 num_colors *= 2 2760 2761 cm = plt.get_cmap("gist_rainbow") 2762 ax.set_prop_cycle( 2763 "color", [cm(1.0 * i / num_colors) for i in range(num_colors)] 2764 ) 2765 2766 for layer_id in range(len(self.neuron_module_vector)): 2767 ax.plot( 2768 np.arange(len(self.member_vars["best_scores"][layer_id])), 2769 self.member_vars["best_scores"][layer_id], 2770 label=self.neuron_module_vector[layer_id].name, 2771 ) 2772 2773 pd2 = pd.DataFrame( 2774 { 2775 "Epochs": np.arange( 2776 len(self.member_vars["best_scores"][layer_id]) 2777 ), 2778 f"Best ever for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 2779 "best_scores" 2780 ][ 2781 layer_id 2782 ], 2783 } 2784 ) 2785 2786 if pd1 is None: 2787 pd1 = pd2 2788 else: 2789 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2790 2791 if len(self.member_vars["current_scores"][layer_id]) != 0: 2792 ax.plot( 2793 np.arange(len(self.member_vars["current_scores"][layer_id])), 2794 self.member_vars["current_scores"][layer_id], 2795 label=f"Current:{self.neuron_module_vector[layer_id].name}", 2796 ) 2797 2798 pd2 = pd.DataFrame( 2799 { 2800 "Epochs": np.arange( 2801 len(self.member_vars["current_scores"][layer_id]) 2802 ), 2803 f"Best current for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 2804 "current_scores" 2805 ][ 2806 layer_id 2807 ], 2808 } 2809 ) 2810 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2811 2812 plt.title(save_folder + "/" + self.save_name + " Best PBScores") 2813 plt.xlabel("Epochs") 2814 plt.ylabel("Best PBScore") 2815 ax.legend( 2816 bbox_to_anchor=(1.05, 1), 2817 loc="upper left", 2818 ncol=max(1, math.ceil(len(self.neuron_module_vector) / 30)), 2819 ) 2820 for switcher in self.member_vars["p_switch_epochs"]: 2821 plt.axvline(x=switcher, ymin=0, ymax=1, color="r") 2822 2823 if self.member_vars["mode"] == "p": 2824 missed_time = ( 2825 self.member_vars["num_epochs_run"] 2826 - self.member_vars["epoch_last_improved"] 2827 ) 2828 plt.axvline( 2829 x=(len(self.member_vars["best_scores"][0]) - (missed_time + 1)), 2830 ymin=0, 2831 ymax=1, 2832 color="g", 2833 ) 2834 2835 # pd1 here will be none if no PB layers are created 2836 if pd1 is not None: 2837 pd1.to_csv( 2838 save_folder 2839 + "/" 2840 + self.save_name 2841 + extra_string 2842 + "Best PBScores.csv", 2843 index=False, 2844 ) 2845 del pd1, pd2 2846 2847 def generate_extra_csv_files(self, save_folder, extra_string): 2848 """ 2849 Generate additional csvs 2850 2851 Parameters 2852 ---------- 2853 save_folder : str 2854 The folder to save the plots and csvs in. 2855 extra_string : str 2856 An extra string to append to the filenames. 2857 2858 Returns 2859 ------- 2860 None 2861 2862 """ 2863 pd1 = pd.DataFrame( 2864 { 2865 "Switch Number": np.arange(len(self.member_vars["switch_epochs"])), 2866 "Switch Epoch": self.member_vars["switch_epochs"], 2867 } 2868 ) 2869 pd1.to_csv( 2870 save_folder + "/" + self.save_name + extra_string + "switch_epochs.csv", 2871 index=False, 2872 ) 2873 del pd1 2874 2875 pd1 = pd.DataFrame( 2876 { 2877 "Switch Number": np.arange(len(self.member_vars["param_counts"])), 2878 "Param Count": self.member_vars["param_counts"], 2879 } 2880 ) 2881 pd1.to_csv( 2882 save_folder + "/" + self.save_name + extra_string + "param_counts.csv", 2883 index=False, 2884 ) 2885 del pd1 2886 2887 """ 2888 Create best_arch_scores.csv file 2889 When working with dendrites there is a tradeoff between additional param count and score improvement. 2890 This file will help track that tradeoff by recording the best scores for all extra_scores 2891 and extra_scores_without_graphing for each architecture version. 2892 The scores recorded here are from the epoch when the best validation score was found 2893 within each switch_epoch boundary. 2894 """ 2895 switch_counts = len(self.member_vars["switch_epochs"]) 2896 best_valid = [] 2897 associated_params = [] 2898 2899 # Initialize dictionaries to store best scores for each extra score type 2900 best_extra_scores = {} 2901 for score_name in self.member_vars["extra_scores"]: 2902 best_extra_scores[score_name] = [] 2903 for score_name in self.member_vars["extra_scores_without_graphing"]: 2904 best_extra_scores[score_name] = [] 2905 2906 for switch in range(0, switch_counts, 2): 2907 start_index = 0 2908 if switch != 0: 2909 start_index = self.member_vars["switch_epochs"][switch - 1] + 1 2910 end_index = self.member_vars["switch_epochs"][switch] + 1 2911 2912 if GPA.pai_tracker.member_vars["maximizing_score"]: 2913 best_valid_index = start_index + np.argmax( 2914 self.member_vars["accuracies"][start_index:end_index] 2915 ) 2916 else: 2917 best_valid_index = start_index + np.argmin( 2918 self.member_vars["accuracies"][start_index:end_index] 2919 ) 2920 2921 best_valid_score = self.member_vars["accuracies"][best_valid_index] 2922 best_valid.append(best_valid_score) 2923 2924 # Get corresponding scores from all extra_scores 2925 for score_name in self.member_vars["extra_scores"]: 2926 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 2927 best_extra_scores[score_name].append( 2928 self.member_vars["extra_scores"][score_name][best_valid_index] 2929 ) 2930 else: 2931 best_extra_scores[score_name].append(None) 2932 2933 # Get corresponding scores from all extra_scores_without_graphing 2934 for score_name in self.member_vars["extra_scores_without_graphing"]: 2935 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 2936 best_extra_scores[score_name].append( 2937 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 2938 ) 2939 else: 2940 best_extra_scores[score_name].append(None) 2941 2942 if self.member_vars["doing_pai"]: 2943 associated_params.append(self.member_vars["param_counts"][switch]) 2944 else: 2945 associated_params.append(self.member_vars["param_counts"][-1]) 2946 2947 # If in neuron training mode but not the very first epoch 2948 if self.member_vars["mode"] == "n" and ( 2949 (len(self.member_vars["switch_epochs"]) == 0) 2950 or ( 2951 self.member_vars["switch_epochs"][-1] + 1 2952 != len(self.member_vars["accuracies"]) 2953 ) 2954 ): 2955 start_index = 0 2956 if len(self.member_vars["switch_epochs"]) != 0: 2957 start_index = self.member_vars["switch_epochs"][-1] + 1 2958 2959 if GPA.pai_tracker.member_vars["maximizing_score"]: 2960 best_valid_index = start_index + np.argmax( 2961 self.member_vars["accuracies"][start_index:] 2962 ) 2963 else: 2964 best_valid_index = start_index + np.argmin( 2965 self.member_vars["accuracies"][start_index:] 2966 ) 2967 2968 best_valid_score = self.member_vars["accuracies"][best_valid_index] 2969 best_valid.append(best_valid_score) 2970 2971 # Get corresponding scores from all extra_scores 2972 for score_name in self.member_vars["extra_scores"]: 2973 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 2974 best_extra_scores[score_name].append( 2975 self.member_vars["extra_scores"][score_name][best_valid_index] 2976 ) 2977 else: 2978 best_extra_scores[score_name].append(None) 2979 2980 # Get corresponding scores from all extra_scores_without_graphing 2981 for score_name in self.member_vars["extra_scores_without_graphing"]: 2982 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 2983 best_extra_scores[score_name].append( 2984 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 2985 ) 2986 else: 2987 best_extra_scores[score_name].append(None) 2988 2989 associated_params.append(self.member_vars["param_counts"][-1]) 2990 2991 # Build dataframe with all columns 2992 csv_data = { 2993 "Param Counts": associated_params, 2994 "Max Valid Scores": best_valid, 2995 } 2996 2997 # Add columns for each extra score 2998 for score_name in best_extra_scores: 2999 csv_data[score_name] = best_extra_scores[score_name] 3000 3001 pd1 = pd.DataFrame(csv_data) 3002 pd1.to_csv( 3003 save_folder + "/" + self.save_name + extra_string + "_best_arch_scores.csv", 3004 index=False, 3005 ) 3006 del pd1 3007 3008 def save_graphs(self, extra_string=""): 3009 """ 3010 Save graphs and csvs for all the values the tracker records 3011 3012 Parameters 3013 ---------- 3014 extra_string : str 3015 An extra string to append to the filenames. 3016 3017 Returns 3018 ------- 3019 None 3020 3021 """ 3022 # If running DDP only save with rank 0 3023 if "RANK" in os.environ: 3024 if int(os.environ["RANK"]) != 0: 3025 return 3026 if not self.making_graphs: 3027 return 3028 3029 save_folder = "./" + self.save_name + "/" 3030 3031 plt.ioff() 3032 fig = plt.figure(figsize=(28, 14)) 3033 3034 # Plot with accuracy scores 3035 ax = plt.subplot(221) 3036 self.generate_accuracy_plots(ax, save_folder, extra_string) 3037 3038 # Plot dendrite learning scores 3039 ax = plt.subplot(222) 3040 self.generate_dendrite_learning_plots(ax, save_folder, extra_string) 3041 3042 if GPA.pc.get_drawing_extra_graphs(): 3043 # Plot learning rates for each training epoch 3044 ax = plt.subplot(223) 3045 self.generate_learning_rate_plots(ax, save_folder, extra_string) 3046 3047 # Plot the times for each training epoch 3048 ax = plt.subplot(224) 3049 self.generate_time_plots(ax, save_folder, extra_string) 3050 3051 # Generate extra CSV files 3052 self.generate_extra_csv_files(save_folder, extra_string) 3053 3054 fig.tight_layout() 3055 plt.savefig(save_folder + "/" + self.save_name + extra_string + ".png") 3056 plt.close("all") 3057 3058 def add_loss(self, loss): 3059 """Add loss to tracking vectors. 3060 3061 Parameters 3062 ---------- 3063 loss : float or int 3064 The loss value to add. 3065 3066 Returns 3067 ------- 3068 None 3069 3070 """ 3071 if not isinstance(loss, (float, int)): 3072 loss = loss.item() 3073 self.member_vars["training_loss"].append(loss) 3074 3075 def add_learning_rate(self, learning_rate): 3076 """Add learning rate to tracking vectors. 3077 3078 Parameters 3079 ---------- 3080 learning_rate : float or int 3081 The learning rate value to add. 3082 3083 Returns 3084 ------- 3085 None 3086 3087 """ 3088 if not isinstance(learning_rate, (float, int)): 3089 learning_rate = learning_rate.item() 3090 self.member_vars["training_learning_rates"].append(learning_rate) 3091 3092 def add_extra_score(self, score, extra_score_name): 3093 """Add extra score to tracking vectors. 3094 3095 Parameters 3096 ---------- 3097 score : float or int 3098 The score value to add. 3099 3100 extra_score_name : str 3101 The name of the extra score. 3102 3103 Returns 3104 ------- 3105 None 3106 3107 """ 3108 if not isinstance(score, (float, int)): 3109 try: 3110 score = score.item() 3111 except: 3112 print( 3113 "Scores added for Perforated Backpropagation should be " 3114 "float, int, or tensor, yours is a:" 3115 ) 3116 print(type(score)) 3117 pdb.set_trace() 3118 3119 if GPA.pc.get_verbose(): 3120 print(f"Adding extra score {extra_score_name} of {float(score)}") 3121 3122 if extra_score_name not in self.member_vars["extra_scores"]: 3123 self.member_vars["extra_scores"][extra_score_name] = [] 3124 self.member_vars["extra_scores"][extra_score_name].append(score) 3125 3126 if self.member_vars["mode"] == "n": 3127 if extra_score_name not in self.member_vars["n_extra_scores"]: 3128 self.member_vars["n_extra_scores"][extra_score_name] = [] 3129 self.member_vars["n_extra_scores"][extra_score_name].append(score) 3130 3131 def add_extra_score_without_graphing(self, score, extra_score_name): 3132 """Add extra score without graphing to tracking vectors. 3133 3134 Parameters 3135 ---------- 3136 score : float or int 3137 The score value to add. 3138 3139 extra_score_name : str 3140 The name of the extra score. 3141 3142 Returns 3143 ------- 3144 None 3145 3146 """ 3147 if not isinstance(score, (float, int)): 3148 try: 3149 score = score.item() 3150 except: 3151 print( 3152 "Scores added for Perforated Backpropagation should be " 3153 "float, int, or tensor, yours is a:" 3154 ) 3155 print(type(score)) 3156 print("in add_extra_score_without_graphing") 3157 pdb.set_trace() 3158 3159 if GPA.pc.get_verbose(): 3160 print(f"Adding extra score {extra_score_name} of {float(score)}") 3161 3162 if extra_score_name not in self.member_vars["extra_scores_without_graphing"]: 3163 self.member_vars["extra_scores_without_graphing"][extra_score_name] = [] 3164 self.member_vars["extra_scores_without_graphing"][extra_score_name].append( 3165 score 3166 ) 3167 3168 def add_test_score(self, score, extra_score_name): 3169 """Add test score to tracking vectors. 3170 3171 Parameters 3172 ---------- 3173 score : float or int 3174 The score value to add. 3175 3176 extra_score_name : str 3177 The name of the extra score. 3178 3179 Returns 3180 ------- 3181 None 3182 3183 Notes 3184 ----- 3185 This function is a wrapper around `add_extra_score` that separates 3186 test score for adding to best_arch_scores.csv. 3187 3188 """ 3189 self.add_extra_score(score, extra_score_name) 3190 3191 if not isinstance(score, (float, int)): 3192 try: 3193 score = score.item() 3194 except: 3195 print( 3196 "Scores added for Perforated Backpropagation should be " 3197 "float, int, or tensor, yours is a:" 3198 ) 3199 print(type(score)) 3200 print("in add_test_score") 3201 pdb.set_trace() 3202 3203 if GPA.pc.get_verbose(): 3204 print(f"Adding test score {extra_score_name} of {float(score)}") 3205 self.member_vars["test_scores"].append(score) 3206 3207 def add_validation_score(self, accuracy, net, force_switch=False): 3208 """Function to add the validation score. 3209 3210 This is complex because it determines neuron and dendrite switching. 3211 3212 Parameters 3213 ---------- 3214 accuracy : float or int 3215 The accuracy or loss value to add. 3216 net : object 3217 The neural network model. 3218 force_switch : bool, optional 3219 Whether to force a switch, by default False. 3220 3221 Returns 3222 ------- 3223 net : object 3224 The potentially modified neural network model. 3225 training_complete : bool 3226 Whether training is complete. 3227 restructured : bool 3228 Whether the model has been restructured. 3229 3230 Notes 3231 ----- 3232 WARNING: Do not call self anywhere in this function. When systems 3233 get loaded the actual tracker you are working with can change. 3234 """ 3235 3236 if not GPA.pc.get_silent(): 3237 print(f"Adding validation score {accuracy:.8f}") 3238 3239 update_learning_rate() 3240 update_param_count(net) 3241 3242 accuracy = check_input_problems(net, accuracy) 3243 3244 if len(GPA.pai_tracker.member_vars["switch_epochs"]) == 0: 3245 epochs_since_cycle_switch = GPA.pai_tracker.member_vars["num_epochs_run"] 3246 else: 3247 epochs_since_cycle_switch = ( 3248 GPA.pai_tracker.member_vars["num_epochs_run"] 3249 - GPA.pai_tracker.member_vars["switch_epochs"][-1] 3250 ) 3251 3252 update_running_accuracy(accuracy, epochs_since_cycle_switch) 3253 if GPA.pc.get_perforated_backpropagation(): 3254 TPB.update_pb_scores(self) 3255 3256 GPA.pai_tracker.stop_epoch(internal_call=True) 3257 3258 # If it is neuron training mode 3259 if ( 3260 GPA.pai_tracker.member_vars["mode"] == "n" 3261 or GPA.pc.get_learn_dendrites_live() 3262 ): 3263 check_new_best(net, accuracy, epochs_since_cycle_switch) 3264 elif GPA.pc.get_perforated_backpropagation(): 3265 TPB.check_best_pai_score_improvement() 3266 3267 # Save the latest model 3268 if GPA.pc.get_test_saves(): 3269 UPA.save_system(net, GPA.pc.get_save_name(), "latest") 3270 if GPA.pc.get_pai_saves(): 3271 UPA.pai_save_system(net, GPA.pc.get_save_name(), "latest") 3272 3273 restructuring_status_value = NO_MODEL_UPDATE 3274 # If it is time to switch based on scores and counter or a manual switch 3275 if GPA.pai_tracker.switch_time() or force_switch: 3276 # If testing dendrite capacity switch after enough dendrites added 3277 if ( 3278 (GPA.pai_tracker.member_vars["mode"] == "n") 3279 and (GPA.pai_tracker.member_vars["num_dendrites_added"] > 2) 3280 and GPA.pc.get_testing_dendrite_capacity() 3281 ): 3282 GPA.pai_tracker.save_graphs() 3283 print( 3284 "Successfully added 3 dendrites with " 3285 "GPA.pc.set_testing_dendrite_capacity(True) (default). " 3286 "You may now set that to False and run a real experiment." 3287 ) 3288 return net, False, True 3289 3290 # If doing neuron training but this dendrite count didn't improve 3291 if ( 3292 (GPA.pai_tracker.member_vars["mode"] == "n") 3293 or GPA.pc.get_learn_dendrites_live() 3294 ) and (GPA.pai_tracker.member_vars["current_n_set_global_best"] is False): 3295 new_restructuring_status_value, net = process_no_improvement(net) 3296 # if this was the final try return that training is complete 3297 if new_restructuring_status_value == TRAINING_COMPLETE: 3298 return net, True, True 3299 else: 3300 restructuring_status_value = update_restructuring_status( 3301 restructuring_status_value, new_restructuring_status_value 3302 ) 3303 # Else if did improve, do a normal switch process 3304 else: 3305 if GPA.pc.get_verbose(): 3306 print( 3307 f"Calling switch_mode with " 3308 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]}, ' 3309 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 3310 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]}, ' 3311 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_value"]},' 3312 f'{GPA.pc.get_max_dendrites()},' 3313 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]},' 3314 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"]},' 3315 ) 3316 import pdb; pdb.set_trace 3317 # If the max number of dendrites has been hit or not doing pai and adding dendtites 3318 # then return rather than adding more 3319 if ( 3320 (GPA.pai_tracker.member_vars["mode"] == "n") 3321 and ( 3322 GPA.pc.get_max_dendrites() 3323 == GPA.pai_tracker.member_vars["num_dendrites_added"] 3324 ) 3325 ) or (GPA.pai_tracker.member_vars["doing_pai"] is False): 3326 if GPA.pc.get_verbose(): 3327 print( 3328 "Max dendrites reached or not doing PAI, finishing training" 3329 ) 3330 net = process_final_network(net) 3331 # Increment integrated if we have dendrites (means they're integrated) 3332 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3333 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3334 if not GPA.pc.get_silent(): 3335 print(f"Final dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3336 return net, True, True 3337 3338 # Otherwise if its neuron training mode reset the counter of failed dendrites 3339 # Check if we should increment integrated count BEFORE change_learning_modes loads old state 3340 should_increment_integrated = False 3341 if GPA.pai_tracker.member_vars["mode"] == "n": 3342 GPA.pai_tracker.member_vars["num_dendrite_tries"] = 0 3343 if GPA.pc.get_verbose(): 3344 print( 3345 "Adding new dendrites without resetting which means " 3346 "the last ones improved. Resetting num_dendrite_tries" 3347 ) 3348 # Remember to increment after change_learning_modes (which loads old tracker state) 3349 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3350 should_increment_integrated = True 3351 3352 GPA.pai_tracker.save_graphs( 3353 f'_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}' 3354 ) 3355 3356 if GPA.pc.get_test_saves(): 3357 UPA.save_system( 3358 net, 3359 GPA.pc.get_save_name(), 3360 f'beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3361 ) 3362 # Copy current best model from this set of dendrites 3363 # If running DDP only copy with rank 0 3364 if "RANK" not in os.environ or int(os.environ["RANK"]) == 0: 3365 shutil.copyfile( 3366 f"{GPA.pc.get_save_name()}/best_model.pt", 3367 f'{GPA.pc.get_save_name()}/best_model_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}.pt', 3368 ) 3369 3370 net = UPA.change_learning_modes( 3371 net, 3372 GPA.pc.get_save_name(), 3373 "best_model", 3374 GPA.pai_tracker.member_vars["doing_pai"], 3375 ) 3376 restructuring_status_value = NETWORK_RESTRUCTURED 3377 3378 # Now increment after change_learning_modes has loaded the best model 3379 # This ensures the increment persists and doesn't get overwritten 3380 if should_increment_integrated: 3381 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3382 if not GPA.pc.get_silent(): 3383 print(f"Dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3384 3385 # If restructured is true, clear scheduler/optimizer before saving 3386 if restructuring_status_value != NETWORK_RESTRUCTURED: 3387 print( 3388 "Restructured should always be triggered here, let us know if you encounter this situation" 3389 ) 3390 pdb.set_trace() 3391 3392 # Since there is a restructuring optimizer and scheduler must be reinitialized after return 3393 GPA.pai_tracker.clear_optimizer_and_scheduler() 3394 3395 # Save the model from after the switch 3396 UPA.save_system( 3397 net, 3398 GPA.pc.get_save_name(), 3399 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3400 ) 3401 3402 # If not time to switch and you have a scheduler, perform the update step 3403 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 3404 new_restructuring_status_value, net = process_scheduler_update( 3405 net, accuracy, epochs_since_cycle_switch 3406 ) 3407 restructuring_status_value = update_restructuring_status( 3408 restructuring_status_value, new_restructuring_status_value 3409 ) 3410 3411 GPA.pai_tracker.start_epoch(internal_call=True) 3412 GPA.pai_tracker.save_graphs() 3413 3414 if restructuring_status_value == NETWORK_RESTRUCTURED: 3415 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 3416 GPA.pai_tracker.member_vars["num_epochs_run"] 3417 ) 3418 if GPA.pc.get_verbose(): 3419 print( 3420 f"Setting epoch last improved to " 3421 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]}' 3422 ) 3423 3424 now = datetime.now() 3425 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 3426 3427 if GPA.pc.get_verbose(): 3428 print("Not saving restructure right now") 3429 3430 """ 3431 This block of code helped with a save issue with safetensors and huggingface, but it breaks DDP. 3432 Temporarily removing it to avoid DDP issues, but if you encounter save issues try adding it back in. 3433 for param in net.parameters(): 3434 param.data = param.data.contiguous() 3435 """ 3436 if GPA.pc.get_verbose(): 3437 print( 3438 f"Completed adding score. Restructured is {restructuring_status_value}, " 3439 f"\ncurrent switch list is:" 3440 ) 3441 print(GPA.pai_tracker.member_vars["switch_epochs"]) 3442 3443 # Always False for training complete if nothing triggered that training is over 3444 return net, restructuring_status_value, False 3445 3446 def clear_all_processors(self): 3447 """Clear all processors from modules.""" 3448 for module in self.neuron_module_vector: 3449 module.clear_processors() 3450 3451 def create_new_dendrite_module(self): 3452 """Add dendrite module to all neuron modules.""" 3453 for module in self.neuron_module_vector: 3454 module.create_new_dendrite_module() 3455 3456 def apply_pb_grads(self): 3457 """Apply perforated backpropagation gradients to all modules.""" 3458 if self.member_vars["mode"] == "p": 3459 for module in self.neuron_module_vector: 3460 module.apply_pb_grads() 3461 3462 def apply_pb_zero(self): 3463 """Apply perforated backpropagation zero gradients to all modules.""" 3464 if self.member_vars["mode"] == "p": 3465 for module in self.neuron_module_vector: 3466 module.apply_pb_zero()
Manager class that tracks all neuron layers and dendrite layers, controls when new dendrites are added, and communicates signals to modules.
PAINeuronModuleTracker( doing_pai, save_name, making_graphs=True, param_vals_setting=-1, values_per_train_epoch=-1, values_per_val_epoch=-1)
931 def __init__( 932 self, 933 doing_pai, 934 save_name, 935 making_graphs=True, 936 param_vals_setting=-1, 937 values_per_train_epoch=-1, 938 values_per_val_epoch=-1, 939 ): 940 """Initialize the tracker 941 942 Parameters 943 ---------- 944 doing_pai : bool 945 Whether or not dendrites should be used. 946 save_name : str 947 The base name for saving models and graphs. 948 making_graphs : bool, optional 949 Whether or not to generate graphs, by default True. 950 param_vals_setting : int, optional 951 Parameter values setting, by default -1. 952 values_per_train_epoch : int, optional 953 The number of values to look back for graphing 954 during training, by default -1 (all values). 955 values_per_val_epoch : int, optional 956 The number of values to look back for graphing 957 during validation, by default -1 (all values). 958 Returns 959 ------- 960 None 961 """ 962 963 # Dict of member vars and their types for saving 964 self.member_vars = {} 965 self.member_var_types = {} 966 967 # Whether or not PAI will be running 968 self.member_vars["doing_pai"] = doing_pai 969 self.member_var_types["doing_pai"] = "bool" 970 971 # How many Dendrites have been added 972 self.member_vars["num_dendrites_added"] = 0 973 self.member_var_types["num_dendrites_added"] = "int" 974 975 # How many Dendrites have been successfully integrated (kept) 976 self.member_vars["num_dendrites_integrated"] = 0 977 self.member_var_types["num_dendrites_integrated"] = "int" 978 979 # How many cycles have been run, *2 or *2+1 of the above 980 self.member_vars["num_cycles"] = 0 981 self.member_var_types["num_cycles"] = "int" 982 983 # Pointers to all neuron wrapped modules 984 self.neuron_module_vector = [] 985 986 # Pointers to all non neuron modules for tracking 987 self.tracked_neuron_module_vector = [] 988 989 # Neuron training or dendrite training mode 990 self.member_vars["mode"] = "n" 991 self.member_var_types["mode"] = "string" 992 993 # Number of epochs run excluding overwritten epochs 994 self.member_vars["num_epochs_run"] = -1 995 self.member_var_types["num_epochs_run"] = "int" 996 997 # Number including overwritten epochs 998 self.member_vars["total_epochs_run"] = -1 999 self.member_var_types["total_epochs_run"] = "int" 1000 1001 # Last epoch that validation/correlation score was improved 1002 self.member_vars["epoch_last_improved"] = 0 1003 self.member_var_types["epoch_last_improved"] = "int" 1004 1005 # Running validation accuracy 1006 self.member_vars["running_accuracy"] = 0 1007 self.member_var_types["running_accuracy"] = "float" 1008 1009 # True if maxing validation, False if minimizing Loss 1010 self.member_vars["maximizing_score"] = True 1011 self.member_var_types["maximizing_score"] = "bool" 1012 1013 # Mode for switching back and forth between learning modes 1014 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 1015 self.member_var_types["switch_mode"] = "int" 1016 1017 # Epoch of the last switch 1018 self.member_vars["last_switch"] = 0 1019 self.member_var_types["last_switch"] = "int" 1020 1021 # Highest validation score from current cycle 1022 self.member_vars["current_best_validation_score"] = 0 1023 self.member_var_types["current_best_validation_score"] = "float" 1024 1025 # Last epoch where the learning rate was updated 1026 self.member_vars["initial_lr_test_epoch_count"] = -1 1027 self.member_var_types["initial_lr_test_epoch_count"] = "int" 1028 1029 # Highest validation score of full run 1030 self.member_vars["global_best_validation_score"] = 0 1031 self.member_var_types["global_best_validation_score"] = "float" 1032 1033 # List of switch epochs 1034 self.member_vars["switch_epochs"] = [] 1035 self.member_var_types["switch_epochs"] = "int array" 1036 1037 # Parameter counts at each network structure 1038 self.member_vars["param_counts"] = [] 1039 self.member_var_types["param_counts"] = "int array" 1040 1041 # List of epochs where switch was made to neuron training 1042 self.member_vars["n_switch_epochs"] = [] 1043 self.member_var_types["n_switch_epochs"] = "int array" 1044 1045 # List of epochs where switch was made to dendrite training 1046 self.member_vars["p_switch_epochs"] = [] 1047 self.member_var_types["p_switch_epochs"] = "int array" 1048 1049 # List of validation accuracies 1050 self.member_vars["accuracies"] = [] 1051 self.member_var_types["accuracies"] = "float array" 1052 1053 # List of epochs where score improved for scheduler updates 1054 self.member_vars["last_improved_accuracies"] = [] 1055 self.member_var_types["last_improved_accuracies"] = "int array" 1056 1057 # List of test accuracy scores registered 1058 self.member_vars["test_accuracies"] = [] 1059 self.member_var_types["test_accuracies"] = "float array" 1060 1061 # List of accuracies registered during neuron training 1062 self.member_vars["n_accuracies"] = [] 1063 self.member_var_types["n_accuracies"] = "float array" 1064 1065 # List of accuracies registered during dendrite training 1066 self.member_vars["p_accuracies"] = [] 1067 self.member_var_types["p_accuracies"] = "float array" 1068 1069 # Running average accuracies from recent epochs 1070 self.member_vars["running_accuracies"] = [] 1071 self.member_var_types["running_accuracies"] = "float array" 1072 1073 # List of additional scores recorded 1074 self.member_vars["extra_scores"] = {} 1075 self.member_var_types["extra_scores"] = "float array dictionary" 1076 1077 # Extra scores not set to be graphed 1078 self.member_vars["extra_scores_without_graphing"] = {} 1079 self.member_var_types["extra_scores_without_graphing"] = ( 1080 "float array dictionary" 1081 ) 1082 1083 # List of test scores 1084 self.member_vars["test_scores"] = [] 1085 self.member_var_types["test_scores"] = "float array" 1086 1087 # Extra scores calculated during neuron training 1088 self.member_vars["n_extra_scores"] = {} 1089 self.member_var_types["n_extra_scores"] = "float array dictionary" 1090 1091 # List of training losses calculated 1092 self.member_vars["training_loss"] = [] 1093 self.member_var_types["training_loss"] = "float array" 1094 1095 # List of learning rates at each epoch 1096 self.member_vars["training_learning_rates"] = [] 1097 self.member_var_types["training_learning_rates"] = "float array" 1098 1099 # Best dendrite scores 1100 self.member_vars["best_scores"] = [] 1101 self.member_var_types["best_scores"] = "float array array" 1102 1103 # Current dendrite scores 1104 self.member_vars["current_scores"] = [] 1105 self.member_var_types["current_scores"] = "float array array" 1106 1107 # Times for neuron training epochs 1108 self.member_vars["n_epoch_times"] = [] 1109 self.member_var_types["n_epoch_times"] = "float array" 1110 1111 # Timing values 1112 self.member_vars["p_epoch_times"] = [] 1113 self.member_var_types["p_epoch_times"] = "float array" 1114 self.member_vars["n_train_times"] = [] 1115 self.member_var_types["n_train_times"] = "float array" 1116 self.member_vars["p_train_times"] = [] 1117 self.member_var_types["p_train_times"] = "float array" 1118 self.member_vars["n_val_times"] = [] 1119 self.member_var_types["n_val_times"] = "float array" 1120 self.member_vars["p_val_times"] = [] 1121 self.member_var_types["p_val_times"] = "float array" 1122 1123 # Setting for tracking timing 1124 self.member_vars["manual_train_switch"] = False 1125 self.member_var_types["manual_train_switch"] = "bool" 1126 1127 # Tracking scores overwritten when reloading best model 1128 self.member_vars["overwritten_extras"] = [] 1129 self.member_var_types["overwritten_extras"] = "float array dictionary array" 1130 self.member_vars["overwritten_vals"] = [] 1131 self.member_var_types["overwritten_vals"] = "float array array" 1132 self.member_vars["overwritten_epochs"] = 0 1133 self.member_var_types["overwritten_epochs"] = "int" 1134 1135 # Setting for determining scores 1136 self.member_vars["param_vals_setting"] = GPA.pc.get_param_vals_setting() 1137 self.member_var_types["param_vals_setting"] = "int" 1138 1139 # Optimizer and scheduler types and instances 1140 self.member_vars["optimizer"] = None 1141 self.member_var_types["optimizer"] = "type" 1142 self.member_vars["scheduler"] = None 1143 self.member_var_types["scheduler"] = "type" 1144 self.member_vars["optimizer_instance"] = None 1145 self.member_var_types["optimizer_instance"] = "empty array" 1146 self.member_vars["scheduler_instance"] = None 1147 self.member_var_types["scheduler_instance"] = "empty array" 1148 1149 # Flag for if the tracker was loaded 1150 self.loaded = False 1151 1152 # flag for 1153 self.member_vars["step_status"] = STEP_CLEARED 1154 self.member_var_types["step_status"] = "int" 1155 1156 1157 # Settings for tracking learning rates 1158 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 1159 self.member_var_types["current_n_learning_rate_initial_skip_steps"] = "int" 1160 self.member_vars["last_max_learning_rate_steps"] = 0 1161 self.member_var_types["last_max_learning_rate_steps"] = "int" 1162 self.member_vars["last_max_learning_rate_value"] = -1 1163 self.member_var_types["last_max_learning_rate_value"] = "float" 1164 self.member_vars["current_cycle_lr_max_scores"] = [] 1165 self.member_var_types["current_cycle_lr_max_scores"] = "float array" 1166 self.member_vars["current_step_count"] = 0 1167 self.member_var_types["current_step_count"] = "int" 1168 self.member_vars["committed_to_initial_rate"] = True 1169 self.member_var_types["committed_to_initial_rate"] = "bool" 1170 self.member_vars["best_mean_score_improved_this_epoch"] = 0 1171 self.member_var_types["best_mean_score_improved_this_epoch"] = "int" 1172 1173 # Flag for if current dendrite achieved highest global score 1174 self.member_vars["current_n_set_global_best"] = True 1175 self.member_var_types["current_n_set_global_best"] = "bool" 1176 1177 # Number of tries adding this dendrite count 1178 self.member_vars["num_dendrite_tries"] = 0 1179 self.member_var_types["num_dendrite_tries"] = "int" 1180 1181 # Count of batches per epoch 1182 self.values_per_train_epoch = values_per_train_epoch 1183 self.values_per_val_epoch = values_per_val_epoch 1184 1185 self.save_name = save_name 1186 self.making_graphs = making_graphs 1187 1188 self.start_time = time.time() 1189 self.saved_time = 0 1190 self.start_epoch(internal_call=True) 1191 1192 if GPA.pc.get_verbose(): 1193 print(f'Initializing with switch_mode {self.member_vars["switch_mode"]}')
Initialize the tracker
Parameters
- doing_pai (bool): Whether or not dendrites should be used.
- save_name (str): The base name for saving models and graphs.
- making_graphs (bool, optional): Whether or not to generate graphs, by default True.
- param_vals_setting (int, optional): Parameter values setting, by default -1.
- values_per_train_epoch (int, optional): The number of values to look back for graphing during training, by default -1 (all values).
- values_per_val_epoch (int, optional): The number of values to look back for graphing during validation, by default -1 (all values).
Returns
- None
def
to_string(self):
1195 def to_string(self): 1196 """Convert tracker values to string for saving with safetensors.""" 1197 1198 full_string = "" 1199 for var in self.member_vars: 1200 full_string += var + "," 1201 if self.member_vars[var] is None: 1202 full_string += "None" 1203 full_string += "\n" 1204 elif self.member_var_types[var] == "bool": 1205 full_string += str(self.member_vars[var]) 1206 full_string += "\n" 1207 elif self.member_var_types[var] in ("int", "float", "string"): 1208 full_string += str(self.member_vars[var]) 1209 full_string += "\n" 1210 elif self.member_var_types[var] == "type": 1211 name = ( 1212 self.member_vars[var].__module__ 1213 + "." 1214 + self.member_vars[var].__name__ 1215 ) 1216 full_string += str(self.member_vars[var]) 1217 full_string += "\n" 1218 elif self.member_var_types[var] == "empty array": 1219 full_string += "[]" 1220 full_string += "\n" 1221 elif self.member_var_types[var] in ("int array", "float array"): 1222 full_string += "\n" 1223 string = "" 1224 for val in self.member_vars[var]: 1225 string += str(val) + "," 1226 # Remove the last comma 1227 string = string[:-1] 1228 full_string += string 1229 full_string += "\n" 1230 elif self.member_var_types[var] == "float array dictionary array": 1231 full_string += "\n" 1232 for array in self.member_vars[var]: 1233 for key in array: 1234 string = key + "," 1235 for val in array[key]: 1236 string += str(val) + "," 1237 # Remove the last comma 1238 string = string[:-1] 1239 full_string += string 1240 full_string += "\n" 1241 full_string += "endkey" 1242 full_string += "\n" 1243 full_string += "endarray" 1244 full_string += "\n" 1245 elif self.member_var_types[var] == "float array dictionary": 1246 full_string += "\n" 1247 for key in self.member_vars[var]: 1248 string = key + "," 1249 for val in self.member_vars[var][key]: 1250 string += str(val) + "," 1251 # Remove the last comma 1252 string = string[:-1] 1253 full_string += string 1254 full_string += "\n" 1255 full_string += "end" 1256 full_string += "\n" 1257 elif self.member_var_types[var] == "float array array": 1258 full_string += "\n" 1259 for array in self.member_vars[var]: 1260 string = "" 1261 for val in array: 1262 string += str(val) + "," 1263 # Remove the last comma 1264 string = string[:-1] 1265 full_string += string 1266 full_string += "\n" 1267 full_string += "end" 1268 full_string += "\n" 1269 else: 1270 print("Did not find a member variable") 1271 pdb.set_trace() 1272 return full_string
Convert tracker values to string for saving with safetensors.
def
from_string(self, string):
1274 def from_string(self, string): 1275 """Load tracker values from string. 1276 1277 Parameters 1278 ---------- 1279 string : str 1280 The string to load from. 1281 """ 1282 f = io.StringIO(string) 1283 while True: 1284 line = f.readline() 1285 if not line: 1286 break 1287 vals = line.split(",") 1288 var = vals[0] 1289 1290 if self.member_var_types[var] == "bool": 1291 val = vals[1][:-1] 1292 if val == "True": 1293 self.member_vars[var] = True 1294 elif val == "False": 1295 self.member_vars[var] = False 1296 elif val == "1": 1297 self.member_vars[var] = 1 1298 elif val == "0": 1299 self.member_vars[var] = 0 1300 else: 1301 print("Something went wrong with loading") 1302 pdb.set_trace() 1303 elif self.member_var_types[var] == "int": 1304 val = vals[1] 1305 self.member_vars[var] = int(val) 1306 elif self.member_var_types[var] == "float": 1307 val = vals[1] 1308 self.member_vars[var] = float(val) 1309 elif self.member_var_types[var] == "string": 1310 val = vals[1][:-1] 1311 self.member_vars[var] = val 1312 elif self.member_var_types[var] == "type": 1313 # Ignore loading types, tracker should have them set up 1314 continue 1315 elif self.member_var_types[var] == "empty array": 1316 val = vals[1] 1317 self.member_vars[var] = [] 1318 elif self.member_var_types[var] == "int array": 1319 vals = f.readline()[:-1].split(",") 1320 self.member_vars[var] = [] 1321 if vals[0] == "": 1322 continue 1323 for val in vals: 1324 self.member_vars[var].append(int(val)) 1325 elif self.member_var_types[var] == "float array": 1326 vals = f.readline()[:-1].split(",") 1327 self.member_vars[var] = [] 1328 if vals[0] == "": 1329 continue 1330 for val in vals: 1331 self.member_vars[var].append(float(val)) 1332 elif self.member_var_types[var] == "float array dictionary array": 1333 self.member_vars[var] = [] 1334 line2 = f.readline()[:-1] 1335 while line2 != "endarray": 1336 temp = {} 1337 while line2 != "endkey": 1338 vals = line2.split(",") 1339 name = vals[0] 1340 temp[name] = [] 1341 vals = vals[1:] 1342 for val in vals: 1343 temp[name].append(float(val)) 1344 line2 = f.readline()[:-1] 1345 self.member_vars[var].append(temp) 1346 line2 = f.readline()[:-1] 1347 elif self.member_var_types[var] == "float array dictionary": 1348 self.member_vars[var] = {} 1349 line2 = f.readline()[:-1] 1350 while line2 != "end": 1351 vals = line2.split(",") 1352 name = vals[0] 1353 self.member_vars[var][name] = [] 1354 vals = vals[1:] 1355 for val in vals: 1356 self.member_vars[var][name].append(float(val)) 1357 line2 = f.readline()[:-1] 1358 elif self.member_var_types[var] == "float array array": 1359 self.member_vars[var] = [] 1360 line2 = f.readline()[:-1] 1361 while line2 != "end": 1362 vals = line2.split(",") 1363 self.member_vars[var].append([]) 1364 if line2: 1365 for val in vals: 1366 self.member_vars[var][-1].append(float(val)) 1367 line2 = f.readline()[:-1] 1368 else: 1369 print("Did not find a member variable") 1370 1371 pdb.set_trace()
Load tracker values from string.
Parameters
- string (str): The string to load from.
def
from_string_debug(self, string):
1373 def from_string_debug(self, string): 1374 """Debug function to print tracker values from string without loading them. 1375 1376 Parameters 1377 ---------- 1378 string : str 1379 The string to debug load from. 1380 """ 1381 f = io.StringIO(string) 1382 print("=== DEBUGGING TRACKER VARIABLES ===") 1383 1384 while True: 1385 line = f.readline() 1386 if not line: 1387 break 1388 vals = line.split(",") 1389 var = vals[0] 1390 1391 print(f"\nVariable: {var}") 1392 print(f"Type: {self.member_var_types.get(var, 'UNKNOWN TYPE')}") 1393 print(f"Current value: {self.member_vars.get(var, 'NOT SET')}") 1394 1395 if self.member_var_types.get(var) == "bool": 1396 val = vals[1][:-1] 1397 print(f"Would set to: {val} -> {val == 'True'}") 1398 1399 elif self.member_var_types.get(var) == "int": 1400 val = vals[1] 1401 print(f"Would set to: {int(val)}") 1402 1403 elif self.member_var_types.get(var) == "float": 1404 val = vals[1] 1405 print(f"Would set to: {float(val)}") 1406 1407 elif self.member_var_types.get(var) == "string": 1408 val = vals[1][:-1] 1409 print(f"Would set to: '{val}'") 1410 1411 elif self.member_var_types.get(var) == "type": 1412 print("Would skip (type loading)") 1413 1414 elif self.member_var_types.get(var) == "empty array": 1415 val = vals[1] 1416 print(f"Would set to: [] (empty array)") 1417 1418 elif self.member_var_types.get(var) == "int array": 1419 vals_line = f.readline()[:-1].split(",") 1420 print(f"Would set to int array with {len(vals_line)} elements:") 1421 if vals_line[0] != "": 1422 print( 1423 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1424 ) 1425 else: 1426 print(" Empty array") 1427 1428 elif self.member_var_types.get(var) == "float array": 1429 vals_line = f.readline()[:-1].split(",") 1430 print(f"Would set to float array with {len(vals_line)} elements:") 1431 if vals_line[0] != "": 1432 print( 1433 f" Elements: {vals_line[:5]}{'...' if len(vals_line) > 5 else ''}" 1434 ) 1435 else: 1436 print(" Empty array") 1437 1438 elif self.member_var_types.get(var) == "float array dictionary array": 1439 print("Would process float array dictionary array:") 1440 array_count = 0 1441 line2 = f.readline()[:-1] 1442 while line2 != "endarray": 1443 key_count = 0 1444 while line2 != "endkey": 1445 vals_dict = line2.split(",") 1446 name = vals_dict[0] 1447 print( 1448 f" Array {array_count}, Key '{name}': {len(vals_dict)-1} elements" 1449 ) 1450 key_count += 1 1451 line2 = f.readline()[:-1] 1452 print(f" Array {array_count} has {key_count} keys") 1453 array_count += 1 1454 line2 = f.readline()[:-1] 1455 print(f" Total arrays: {array_count}") 1456 1457 elif self.member_var_types.get(var) == "float array dictionary": 1458 print("Would process float array dictionary:") 1459 line2 = f.readline()[:-1] 1460 key_count = 0 1461 while line2 != "end": 1462 vals_dict = line2.split(",") 1463 name = vals_dict[0] 1464 print(f" Key '{name}': {len(vals_dict)-1} elements") 1465 key_count += 1 1466 line2 = f.readline()[:-1] 1467 print(f" Total keys: {key_count}") 1468 1469 elif self.member_var_types.get(var) == "float array array": 1470 print("Would process float array array:") 1471 line2 = f.readline()[:-1] 1472 array_count = 0 1473 while line2 != "end": 1474 if line2: 1475 vals_array = line2.split(",") 1476 print(f" Array {array_count}: {len(vals_array)} elements") 1477 else: 1478 print(f" Array {array_count}: empty") 1479 array_count += 1 1480 line2 = f.readline()[:-1] 1481 print(f" Total arrays: {array_count}") 1482 1483 else: 1484 print(f"UNKNOWN TYPE: {self.member_var_types.get(var, 'NOT FOUND')}") 1485 1486 print("\n=== END DEBUG ===")
Debug function to print tracker values from string without loading them.
Parameters
- string (str): The string to debug load from.
def
save_tracker_settings(self):
1488 def save_tracker_settings(self): 1489 """Save tracker settings for DistributedDataParallel use. 1490 1491 Saves settings in save_name/array_dims.csv 1492 1493 Parameters 1494 ---------- 1495 None 1496 Returns 1497 ------- 1498 None 1499 1500 ----- 1501 Instructions for use are in API customization.md 1502 """ 1503 if not os.path.isdir(self.save_name): 1504 os.makedirs(self.save_name) 1505 f = open(self.save_name + "/array_dims.csv", "w") 1506 for layer in self.neuron_module_vector: 1507 f.write( 1508 f"{layer.name},{layer.dendrite_module.dendrite_values[0].out_channels}\n" 1509 ) 1510 f.close() 1511 if not GPA.pc.get_silent(): 1512 print("Tracker settings saved.") 1513 print("You may now delete save_tracker_settings")
Save tracker settings for DistributedDataParallel use.
Saves settings in save_name/array_dims.csv
Parameters
- None
Returns
- None
- -----
- Instructions for use are in API customization.md
def
initialize_tracker_settings(self):
1515 def initialize_tracker_settings(self): 1516 """Initialize tracker settings from saved file. 1517 1518 This function loads tracker settings from a CSV file and applies them 1519 to the layers the tracker is managing. 1520 1521 Parameters 1522 ---------- 1523 None 1524 1525 Returns 1526 ------- 1527 None 1528 1529 """ 1530 1531 channels = {} 1532 if not os.path.exists(self.save_name + "/array_dims.csv"): 1533 print( 1534 "You must call save_tracker_settings before " 1535 "initialize_tracker_settings" 1536 ) 1537 print("Follow instructions in customization.md") 1538 pdb.set_trace() 1539 f = open(self.save_name + "/array_dims.csv", "r") 1540 for line in f: 1541 channels[line.split(",")[0]] = int(line.split(",")[1]) 1542 for layer in self.neuron_module_vector: 1543 layer.dendrite_module.dendrite_values[0].setup_arrays(channels[layer.name])
Initialize tracker settings from saved file.
This function loads tracker settings from a CSV file and applies them to the layers the tracker is managing.
Parameters
- None
Returns
- None
def
set_optimizer_instance(self, optimizer_instance):
1545 def set_optimizer_instance(self, optimizer_instance): 1546 """Set optimizer instance directly. 1547 1548 Parameters 1549 ---------- 1550 optimizer_instance : object 1551 The optimizer instance to set. 1552 1553 Returns 1554 ------- 1555 None 1556 1557 """ 1558 1559 try: 1560 for param_group in optimizer_instance.param_groups: 1561 if ( 1562 param_group["weight_decay"] > 0 1563 and GPA.pc.get_weight_decay_accepted() is False 1564 ): 1565 print( 1566 "For PAI training it is recommended to not use " 1567 "weight decay in your optimizer" 1568 ) 1569 1570 except: 1571 pass 1572 self.member_vars["optimizer_instance"] = optimizer_instance 1573 if GPA.pc.get_perforated_backpropagation(): 1574 TPB.setup_optimizer_pb(self.member_vars["optimizer_instance"])
Set optimizer instance directly.
Parameters
- optimizer_instance (object): The optimizer instance to set.
Returns
- None
def
set_optimizer(self, optimizer):
1576 def set_optimizer(self, optimizer): 1577 """Set optimizer type to be initialized later 1578 1579 Parameters 1580 ---------- 1581 optimizer : object 1582 The optimizer type to set. 1583 1584 Returns 1585 ------- 1586 None 1587 1588 """ 1589 self.member_vars["optimizer"] = optimizer
Set optimizer type to be initialized later
Parameters
- optimizer (object): The optimizer type to set.
Returns
- None
def
set_scheduler(self, scheduler):
1591 def set_scheduler(self, scheduler): 1592 """Set scheduler type to be initialized later 1593 1594 Parameters 1595 ---------- 1596 scheduler : object 1597 The scheduler type to set. 1598 1599 Returns 1600 ------- 1601 None 1602 1603 """ 1604 if scheduler is not torch.optim.lr_scheduler.ReduceLROnPlateau: 1605 if GPA.pc.get_verbose(): 1606 print("Not using ReduceLROnPlateau, this is not recommended") 1607 self.member_vars["scheduler"] = scheduler
Set scheduler type to be initialized later
Parameters
- scheduler (object): The scheduler type to set.
Returns
- None
def
increment_scheduler(self, num_ticks, mode):
1609 def increment_scheduler(self, num_ticks, mode): 1610 """Increment the scheduler a set number of times. 1611 1612 Used for finding best initial learning rate when adding dendrites. 1613 1614 Parameters 1615 ---------- 1616 num_ticks : int 1617 The number of scheduler steps to take. 1618 mode : str 1619 The mode for stepping the scheduler. Options are: 1620 - "step_learning_rate": Step based on improved accuracy epochs 1621 - "increment_epoch_count": Step based on total epoch count 1622 1623 Returns 1624 ------- 1625 current_steps : int 1626 The number of learning rate changes that occurred. 1627 learning_rate1 : float 1628 The final learning rate after stepping. 1629 1630 """ 1631 1632 current_steps = 0 1633 current_ticker = 0 1634 1635 for param_group in GPA.pai_tracker.member_vars[ 1636 "optimizer_instance" 1637 ].param_groups: 1638 learning_rate1 = param_group["lr"] 1639 1640 if GPA.pc.get_verbose(): 1641 print("Using scheduler:") 1642 print(type(self.member_vars["scheduler_instance"])) 1643 1644 while current_ticker < num_ticks: 1645 if GPA.pc.get_verbose(): 1646 print( 1647 f"Lower start rate initial {learning_rate1} " 1648 f'stepping {GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]} times' 1649 ) 1650 1651 if ( 1652 type(self.member_vars["scheduler_instance"]) 1653 is torch.optim.lr_scheduler.ReduceLROnPlateau 1654 ): 1655 if mode == "step_learning_rate": 1656 # Step with counter as last improved accuracy 1657 self.member_vars["scheduler_instance"].step( 1658 metrics=self.member_vars["last_improved_accuracies"][ 1659 GPA.pai_tracker.steps_after_switch() - 1 1660 ] 1661 ) 1662 elif mode == "increment_epoch_count": 1663 # Step with improved epoch counts up to current location 1664 self.member_vars["scheduler_instance"].step( 1665 metrics=self.member_vars["last_improved_accuracies"][ 1666 -((num_ticks - 1) - current_ticker) - 1 1667 ] 1668 ) 1669 else: 1670 self.member_vars["scheduler_instance"].step() 1671 1672 for param_group in GPA.pai_tracker.member_vars[ 1673 "optimizer_instance" 1674 ].param_groups: 1675 learning_rate2 = param_group["lr"] 1676 1677 if learning_rate2 != learning_rate1: 1678 current_steps += 1 1679 learning_rate1 = learning_rate2 1680 if mode == "step_learning_rate": 1681 current_ticker += 1 1682 if GPA.pc.get_verbose(): 1683 print(f"1 step {current_steps} to {learning_rate2}") 1684 1685 if mode == "increment_epoch_count": 1686 current_ticker += 1 1687 1688 return current_steps, learning_rate1
Increment the scheduler a set number of times.
Used for finding best initial learning rate when adding dendrites.
Parameters
- num_ticks (int): The number of scheduler steps to take.
- mode (str):
The mode for stepping the scheduler. Options are:
- "step_learning_rate": Step based on improved accuracy epochs
- "increment_epoch_count": Step based on total epoch count
Returns
- current_steps (int): The number of learning rate changes that occurred.
- learning_rate1 (float): The final learning rate after stepping.
def
setup_optimizer(self, net, opt_args, sched_args=None, parameters=None):
1690 def setup_optimizer(self, net, opt_args, sched_args=None, parameters=None): 1691 """Initialize the optimizer and scheduler when added. 1692 1693 Parameters 1694 ---------- 1695 net : object 1696 The neural network model. 1697 opt_args : dict 1698 The arguments for the optimizer. 1699 sched_args : dict, optional 1700 The arguments for the scheduler, by default None. 1701 1702 Returns 1703 ------- 1704 optimizer : object 1705 The initialized optimizer instance. 1706 scheduler : object, optional 1707 The initialized scheduler instance, if a scheduler was set. 1708 1709 """ 1710 if "weight_decay" in opt_args and not GPA.pc.get_weight_decay_accepted(): 1711 print( 1712 "For PAI training it is recommended to not use " 1713 "weight decay in your optimizer" 1714 ) 1715 1716 if ("model" not in opt_args.keys()) and "params" not in opt_args.keys(): 1717 print("In setup_optimizer it will be depreciated to not pass in params yourself in the future") 1718 print("please change the settings to include params") 1719 if self.member_vars["mode"] == "n": 1720 if parameters is not None: 1721 opt_args["params"] = parameters 1722 else: 1723 opt_args["params"] = filter(lambda p: p.requires_grad, net.parameters()) 1724 else: 1725 params = UPA.get_pai_network_params(net) 1726 if parameters is not None: 1727 # Filter parameters to only those in params, preserving weight_decay 1728 params_set = set(params) 1729 filtered_params = [] 1730 for param_group in parameters: 1731 filtered_group_params = [p for p in param_group["params"] if p in params_set] 1732 if filtered_group_params: 1733 filtered_params.append({ 1734 "params": filtered_group_params, 1735 "weight_decay": param_group["weight_decay"] 1736 }) 1737 opt_args["params"] = filtered_params 1738 else: 1739 opt_args["params"] = params 1740 elif "params" in opt_args.keys(): 1741 # Check if params is a list of param groups (dicts) or a single param group 1742 params_value = opt_args["params"] 1743 if isinstance(params_value, list) and len(params_value) > 0: 1744 # Check if it's a list of dicts (multiple param groups) or list of tensors (single group) 1745 if isinstance(params_value[0], dict): 1746 # Multiple param groups format: [{"params": [...], "lr": ...}, ...] 1747 # Filter each param group for requires_grad 1748 filtered_param_groups = [] 1749 for param_group in params_value: 1750 filtered_group_params = [p for p in param_group["params"] if p.requires_grad] 1751 if filtered_group_params: 1752 new_group = param_group.copy() 1753 new_group["params"] = filtered_group_params 1754 filtered_param_groups.append(new_group) 1755 opt_args["params"] = filtered_param_groups 1756 else: 1757 # Single param group format: [tensor1, tensor2, ...] or generator 1758 # Filter for requires_grad 1759 opt_args["params"] = [p for p in params_value if p.requires_grad] 1760 elif hasattr(params_value, '__iter__'): 1761 # Handle generators or other iterables 1762 opt_args["params"] = [p for p in params_value if p.requires_grad] 1763 1764 optimizer = self.member_vars["optimizer"](**opt_args) 1765 self.set_optimizer_instance(optimizer) 1766 1767 if self.member_vars["scheduler"] is not None: 1768 # Handle SequentialLR specially 1769 if self.member_vars["scheduler"] is torch.optim.lr_scheduler.SequentialLR: 1770 """ 1771 sched_args should be a dict with "schedulers" (list of tuples) and "milestones" 1772 For example: 1773 sequential_schedArgs = { 1774 "schedulers": [ 1775 (warmup_scheduler_class, warmup_schedArgs), 1776 (main_scheduler_class, main_schedArgs) 1777 ], 1778 "milestones": [switch_epoch] 1779 } 1780 """ 1781 schedulers = [] 1782 milestones = sched_args.get("milestones", []) 1783 scheduler_configs = sched_args.get("schedulers", []) 1784 1785 for scheduler_class, scheduler_args in scheduler_configs: 1786 schedulers.append(scheduler_class(optimizer, **scheduler_args)) 1787 1788 self.member_vars["scheduler_instance"] = torch.optim.lr_scheduler.SequentialLR( 1789 optimizer, schedulers=schedulers, milestones=milestones 1790 ) 1791 else: 1792 self.member_vars["scheduler_instance"] = self.member_vars["scheduler"]( 1793 optimizer, **sched_args 1794 ) 1795 current_steps = 0 1796 1797 for param_group in GPA.pai_tracker.member_vars[ 1798 "optimizer_instance" 1799 ].param_groups: 1800 learning_rate1 = param_group["lr"] 1801 1802 if GPA.pc.get_verbose(): 1803 print( 1804 f"Resetting scheduler with {GPA.pai_tracker.steps_after_switch()} " 1805 f"steps and {GPA.pc.get_initial_history_after_switches()} initial ticks to skip" 1806 ) 1807 1808 # Find setting of previously used learning rate before adding dendrites 1809 if ( 1810 GPA.pai_tracker.member_vars[ 1811 "current_n_learning_rate_initial_skip_steps" 1812 ] 1813 != 0 1814 ): 1815 additional_steps, learning_rate1 = self.increment_scheduler( 1816 GPA.pai_tracker.member_vars[ 1817 "current_n_learning_rate_initial_skip_steps" 1818 ], 1819 "step_learning_rate", 1820 ) 1821 current_steps += additional_steps 1822 1823 if self.member_vars["mode"] == "n" or GPA.pc.get_learn_dendrites_live(): 1824 initial = GPA.pc.get_initial_history_after_switches() 1825 else: 1826 initial = 0 1827 1828 if GPA.pai_tracker.steps_after_switch() > initial: 1829 # Minus extra 1 because this gets called after start epoch 1830 additional_steps, learning_rate1 = self.increment_scheduler( 1831 (GPA.pai_tracker.steps_after_switch() - initial) - 1, 1832 "increment_epoch_count", 1833 ) 1834 current_steps += additional_steps 1835 1836 if GPA.pc.get_verbose(): 1837 print( 1838 f"Scheduler update loop with {current_steps} " 1839 f"ended with {learning_rate1}" 1840 ) 1841 print( 1842 f"Scheduler ended with {current_steps} steps " 1843 f"and lr of {learning_rate1}" 1844 ) 1845 1846 self.member_vars["current_step_count"] = current_steps 1847 return optimizer, self.member_vars["scheduler_instance"] 1848 else: 1849 return optimizer
Initialize the optimizer and scheduler when added.
Parameters
- net (object): The neural network model.
- opt_args (dict): The arguments for the optimizer.
- sched_args (dict, optional): The arguments for the scheduler, by default None.
Returns
- optimizer (object): The initialized optimizer instance.
- scheduler (object, optional): The initialized scheduler instance, if a scheduler was set.
def
clear_optimizer_and_scheduler(self):
1851 def clear_optimizer_and_scheduler(self): 1852 """Clear the instances for saving.""" 1853 self.member_vars["optimizer_instance"] = None 1854 self.member_vars["scheduler_instance"] = None
Clear the instances for saving.
def
switch_time(self):
1856 def switch_time(self): 1857 """Determine if it's time to switch between neuron and dendrite training. 1858 1859 Parameters 1860 ---------- 1861 None 1862 1863 Returns 1864 ------- 1865 bool 1866 True if it's time to switch, False otherwise. 1867 1868 Notes 1869 ----- 1870 Based on current settings and history of scores. 1871 """ 1872 1873 switch_phrase = "No mode, this should never be the case." 1874 switch_number = GPA.pc.get_n_epochs_to_switch() 1875 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1876 switch_phrase = "DOING_SWITCH_EVERY_TIME" 1877 elif self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY: 1878 switch_phrase = "DOING_HISTORY" 1879 elif self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH: 1880 switch_phrase = "DOING_FIXED_SWITCH" 1881 switch_number = GPA.pc.get_fixed_switch_num() 1882 elif self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1883 switch_phrase = "DOING_NO_SWITCH" 1884 else: 1885 print( 1886 "A switch mode must be set. Check your settings for GPA.pc.set_switch_mode()." 1887 ) 1888 pdb.set_trace() 1889 if not GPA.pc.get_silent(): 1890 if(GPA.pc.get_perforated_backpropagation()): 1891 print( 1892 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1893 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1894 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1895 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1896 f'n: {switch_number}, p: {GPA.pc.get_p_epochs_to_switch()}, ' 1897 f'num_cycles: {self.member_vars["num_cycles"]}' 1898 ) 1899 else: 1900 print( 1901 f'Checking PAI switch with mode {self.member_vars["mode"]}, ' 1902 f'switch mode {switch_phrase}, epoch {self.member_vars["num_epochs_run"]}, ' 1903 f'last improved epoch {self.member_vars["epoch_last_improved"]}, ' 1904 f'total epochs {self.member_vars["total_epochs_run"]}, ' 1905 f'n: {switch_number}, num_cycles: {self.member_vars["num_cycles"]}' 1906 ) 1907 print( 1908 f' Score tracking: current_n_set_global_best={self.member_vars["current_n_set_global_best"]}, ' 1909 f'global_best={self.member_vars["global_best_validation_score"]:.4f}, ' 1910 f'current_best={self.member_vars["current_best_validation_score"]:.4f}' 1911 ) 1912 if GPA.pc.get_perforated_backpropagation(): 1913 # this will fill in epoch last improved 1914 TPB.best_pai_score_improved_this_epoch(self) ## CLOSED ONLY 1915 if self.member_vars["switch_mode"] == GPA.pc.DOING_NO_SWITCH: 1916 if not GPA.pc.get_silent(): 1917 print("Returning False - doing no switch mode") 1918 return False 1919 1920 if self.member_vars["switch_mode"] == GPA.pc.DOING_SWITCH_EVERY_TIME: 1921 if not GPA.pc.get_silent(): 1922 print("Returning True - switching every time") 1923 return True 1924 1925 # Check if we're in the middle of learning rate optimization 1926 # If so, block ALL switch triggers until committed 1927 if GPA.pc.get_verbose(): 1928 print("=== LR Optimization Check ===") 1929 print(f' mode == "n": {self.member_vars["mode"] == "n"}') 1930 print(f" get_learn_dendrites_live(): {GPA.pc.get_learn_dendrites_live()}") 1931 print(f' committed_to_initial_rate: {GPA.pai_tracker.member_vars["committed_to_initial_rate"]}') 1932 print(f" get_dont_give_up_unless_learning_rate_lowered(): {GPA.pc.get_dont_give_up_unless_learning_rate_lowered()}") 1933 print(f' current_n_learning_rate_initial_skip_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"]}') 1934 print(f' last_max_learning_rate_steps: {self.member_vars["last_max_learning_rate_steps"]}') 1935 print(f' skip_steps < max_steps: {self.member_vars["current_n_learning_rate_initial_skip_steps"] < self.member_vars["last_max_learning_rate_steps"]}') 1936 print(f' scheduler is not None: {self.member_vars["scheduler"] is not None}') 1937 print("=============================") 1938 1939 if ( 1940 ((self.member_vars["mode"] == "n") or GPA.pc.get_learn_dendrites_live()) 1941 and (GPA.pai_tracker.member_vars["committed_to_initial_rate"] is False) 1942 and (GPA.pc.get_dont_give_up_unless_learning_rate_lowered()) 1943 and ( 1944 self.member_vars["current_n_learning_rate_initial_skip_steps"] 1945 <= self.member_vars["last_max_learning_rate_steps"] 1946 ) 1947 and self.member_vars["scheduler"] is not None 1948 ): 1949 if not GPA.pc.get_silent(): 1950 print( 1951 f"Returning False - learning rate optimization in progress. " 1952 f"Not committed yet. Comparing " 1953 f'initial {self.member_vars["current_n_learning_rate_initial_skip_steps"]} ' 1954 f'to last max {self.member_vars["last_max_learning_rate_steps"]}' 1955 ) 1956 return False 1957 1958 if len(self.member_vars["switch_epochs"]) == 0: 1959 this_count = self.member_vars["num_epochs_run"] 1960 else: 1961 this_count = ( 1962 self.member_vars["num_epochs_run"] 1963 - self.member_vars["switch_epochs"][-1] 1964 ) 1965 cap_switch = False 1966 if GPA.pc.get_perforated_backpropagation(): 1967 cap_switch = TPB.check_cap_switch(self, this_count) 1968 1969 if self.member_vars["switch_mode"] == GPA.pc.DOING_HISTORY and ( 1970 ( 1971 (self.member_vars["mode"] == "n") 1972 and ( 1973 self.member_vars["num_epochs_run"] 1974 - self.member_vars["epoch_last_improved"] 1975 >= GPA.pc.get_n_epochs_to_switch() 1976 ) 1977 and this_count 1978 >= GPA.pc.get_initial_history_after_switches() 1979 + GPA.pc.get_n_epochs_to_switch() 1980 ) 1981 or (GPA.pc.get_perforated_backpropagation() and TPB.history_switch(self)) 1982 or cap_switch 1983 ): 1984 if not GPA.pc.get_silent(): 1985 print("Returning True - History and last improved is hit") 1986 return True 1987 1988 if self.member_vars["switch_mode"] == GPA.pc.DOING_FIXED_SWITCH and ( 1989 ( 1990 self.member_vars["total_epochs_run"] % GPA.pc.get_fixed_switch_num() 1991 == GPA.pc.get_fixed_switch_num() - 1 1992 ) 1993 and self.member_vars["num_epochs_run"] 1994 >= GPA.pc.get_first_fixed_switch_num() - 1 1995 ): 1996 if not GPA.pc.get_silent(): 1997 print("Returning True - Fixed switch number is hit") 1998 return True 1999 2000 if not GPA.pc.get_silent(): 2001 print("Returning False - no triggers to switch have been hit") 2002 return False
Determine if it's time to switch between neuron and dendrite training.
Parameters
- None
Returns
- bool: True if it's time to switch, False otherwise.
Notes
Based on current settings and history of scores.
def
steps_after_switch(self):
2004 def steps_after_switch(self): 2005 """Based on settings, return value for steps since a switch. 2006 2007 Different options for param vals setting determine what is returned. 2008 2009 Parameters 2010 ---------- 2011 None 2012 2013 Returns 2014 ------- 2015 int 2016 The number of epochs since the last switch, or total epochs run, 2017 depending on settings. 2018 2019 """ 2020 if self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_TOTAL_EPOCH: 2021 return self.member_vars["num_epochs_run"] 2022 elif ( 2023 self.member_vars["param_vals_setting"] == GPA.pc.PARAM_VALS_BY_UPDATE_EPOCH 2024 ): 2025 return self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2026 elif ( 2027 self.member_vars["param_vals_setting"] 2028 == GPA.pc.PARAM_VALS_BY_NEURON_EPOCH_START 2029 ): 2030 if self.member_vars["mode"] == "p": 2031 return ( 2032 self.member_vars["num_epochs_run"] - self.member_vars["last_switch"] 2033 ) 2034 else: 2035 return self.member_vars["num_epochs_run"] 2036 else: 2037 print( 2038 f'{self.member_vars["param_vals_setting"]} is not a valid param vals option' 2039 ) 2040 pdb.set_trace()
Based on settings, return value for steps since a switch.
Different options for param vals setting determine what is returned.
Parameters
- None
Returns
- int: The number of epochs since the last switch, or total epochs run, depending on settings.
def
add_pai_neuron_module(self, new_module, initial_add=True):
2042 def add_pai_neuron_module(self, new_module, initial_add=True): 2043 """Add neuron modules to internal vectors. 2044 2045 Parameters 2046 ---------- 2047 new_module : object 2048 The new module to add. 2049 initial_add : bool, optional 2050 Whether this is the initial addition rather than loading from file 2051 2052 Returns 2053 ------- 2054 None 2055 2056 """ 2057 2058 # If it's a duplicate, ignore the second addition 2059 if new_module in self.neuron_module_vector: 2060 return 2061 self.neuron_module_vector.append(new_module) 2062 if self.member_vars["doing_pai"]: 2063 PA.set_wrapped_params(new_module) 2064 if initial_add: 2065 self.member_vars["best_scores"].append([]) 2066 self.member_vars["current_scores"].append([])
Add neuron modules to internal vectors.
Parameters
- new_module (object): The new module to add.
- initial_add (bool, optional): Whether this is the initial addition rather than loading from file
Returns
- None
def
add_tracked_neuron_module(self, new_module, initial_add=True):
2068 def add_tracked_neuron_module(self, new_module, initial_add=True): 2069 """Add tracked modules to internal vectors 2070 2071 Parameters 2072 ---------- 2073 new_module : object 2074 The new module to add. 2075 initial_add : bool, optional 2076 Whether this is the initial addition rather than loading from file 2077 2078 Returns 2079 ------- 2080 None 2081 2082 """ 2083 # If it's a duplicate, ignore the second addition 2084 if new_module in self.tracked_neuron_module_vector: 2085 return 2086 self.tracked_neuron_module_vector.append(new_module) 2087 if self.member_vars["doing_pai"]: 2088 PA.set_tracked_params(new_module)
Add tracked modules to internal vectors
Parameters
- new_module (object): The new module to add.
- initial_add (bool, optional): Whether this is the initial addition rather than loading from file
Returns
- None
def
reset_module_vector(self, net, load_from_restart):
2090 def reset_module_vector(self, net, load_from_restart): 2091 """Clear internal vectors and reset from network. 2092 2093 Parameters 2094 ---------- 2095 net : object 2096 The neural network model. 2097 load_from_restart : bool 2098 Whether loading from a restart file. 2099 2100 Returns 2101 ------- 2102 None 2103 2104 """ 2105 self.neuron_module_vector = [] 2106 self.tracked_neuron_module_vector = [] 2107 this_list = UPA.get_pai_modules(net, 0) 2108 for module in this_list: 2109 self.add_pai_neuron_module(module, initial_add=load_from_restart) 2110 this_list = UPA.get_tracked_modules(net, 0) 2111 for module in this_list: 2112 self.add_tracked_neuron_module(module, initial_add=load_from_restart)
Clear internal vectors and reset from network.
Parameters
- net (object): The neural network model.
- load_from_restart (bool): Whether loading from a restart file.
Returns
- None
def
reset_vals_for_score_reset(self):
2114 def reset_vals_for_score_reset(self): 2115 """Reset cycle scores for new cycle.""" 2116 2117 if GPA.pc.get_find_best_lr(): 2118 self.member_vars["committed_to_initial_rate"] = False 2119 print("Resetting committed to initial rate to False") 2120 # If retaining all dendrties always say that the current dendrites set global best for saving and loading 2121 if GPA.pc.get_retain_all_dendrites(): 2122 self.member_vars["current_n_set_global_best"] = True 2123 self.member_vars["global_best_validation_score"] = 0 2124 else: 2125 self.member_vars["current_n_set_global_best"] = False 2126 2127 # Don't reset global best, but do reset current best 2128 self.member_vars["current_best_validation_score"] = 0 2129 self.member_vars["initial_lr_test_epoch_count"] = -1
Reset cycle scores for new cycle.
def
set_dendrite_training(self):
2131 def set_dendrite_training(self): 2132 """Signal all layers to start dendrite training.""" 2133 if GPA.pc.get_verbose(): 2134 print("Calling set_dendrite_training") 2135 2136 for layer in self.neuron_module_vector[:]: 2137 worked = layer.set_mode("p") 2138 """ 2139 worked is False when a layer was added to the neuron module vector 2140 but then it's never actually been used. This can happen when 2141 you have set a layer to have requires_grad = False or when 2142 you have a module as a member variable but it's not actually 2143 part of the network. Should be moved to be a tracked layer 2144 rather than a neuron layer. 2145 """ 2146 if not worked: 2147 self.neuron_module_vector.remove(layer) 2148 2149 for layer in self.tracked_neuron_module_vector[:]: 2150 worked = layer.set_mode("p") 2151 2152 self.create_new_dendrite_module() 2153 self.member_vars["mode"] = "p" 2154 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2155 2156 if GPA.pc.get_learn_dendrites_live(): 2157 self.reset_vals_for_score_reset() 2158 2159 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2160 "current_step_count" 2161 ] 2162 2163 GPA.pai_tracker.member_vars["current_cycle_lr_max_scores"] = [] 2164 GPA.pai_tracker.member_vars["num_cycles"] += 1
Signal all layers to start dendrite training.
def
set_neuron_training(self):
2167 def set_neuron_training(self): 2168 """Signal all layers to start neuron training.""" 2169 for module in self.neuron_module_vector: 2170 module.set_mode("n") 2171 for module in self.tracked_neuron_module_vector[:]: 2172 module.set_mode("n") 2173 2174 self.member_vars["mode"] = "n" 2175 self.member_vars["num_dendrites_added"] += 1 2176 self.member_vars["current_n_learning_rate_initial_skip_steps"] = 0 2177 self.reset_vals_for_score_reset() 2178 2179 self.member_vars["current_cycle_lr_max_scores"] = [] 2180 if GPA.pc.get_learn_dendrites_live(): 2181 self.member_vars["last_max_learning_rate_steps"] = self.member_vars[ 2182 "current_step_count" 2183 ] 2184 GPA.pai_tracker.member_vars["num_cycles"] += 1 2185 2186 if GPA.pc.get_reset_best_score_on_switch(): 2187 GPA.pai_tracker.member_vars["current_best_validation_score"] = 0 2188 GPA.pai_tracker.member_vars["running_accuracy"] = 0
Signal all layers to start neuron training.
def
start_epoch(self, internal_call=False):
2190 def start_epoch(self, internal_call=False): 2191 """Perform steps for when a new training epoch is about to begin. 2192 2193 Parameters 2194 ---------- 2195 internal_call : bool, optional 2196 Whether this is an internal call or manual call 2197 2198 Returns 2199 ------- 2200 None 2201 2202 Notes 2203 ----- 2204 If you ever need to call this manually, set internal_call to False. 2205 2206 """ 2207 if self.member_vars["manual_train_switch"] and internal_call: 2208 return 2209 2210 if not internal_call and not self.member_vars["manual_train_switch"]: 2211 self.member_vars["manual_train_switch"] = True 2212 self.saved_time = 0 2213 self.member_vars["num_epochs_run"] = -1 2214 self.member_vars["total_epochs_run"] = -1 2215 2216 end = time.time() 2217 if self.member_vars["manual_train_switch"]: 2218 if self.saved_time != 0: 2219 if self.member_vars["mode"] == "p": 2220 self.member_vars["p_val_times"].append(end - self.saved_time) 2221 else: 2222 self.member_vars["n_val_times"].append(end - self.saved_time) 2223 2224 if self.member_vars["mode"] == "p": 2225 for layer in self.neuron_module_vector: 2226 for m in range(0, GPA.pc.get_global_candidates()): 2227 with torch.no_grad(): 2228 if GPA.pc.get_verbose(): 2229 print(f"Resetting score for {layer.name}") 2230 # Snapshot best_score before reset so we can compute per-epoch improvement 2231 layer.dendrite_module.dendrite_values[ 2232 m 2233 ].epoch_start_best_score.copy_( 2234 layer.dendrite_module.dendrite_values[ 2235 m 2236 ].best_score.detach() 2237 ) 2238 layer.dendrite_module.dendrite_values[ 2239 m 2240 ].best_score_improved_this_epoch = ( 2241 layer.dendrite_module.dendrite_values[ 2242 m 2243 ].best_score_improved_this_epoch 2244 * 0 2245 ) 2246 layer.dendrite_module.dendrite_values[ 2247 m 2248 ].nodes_best_improved_this_epoch = ( 2249 layer.dendrite_module.dendrite_values[ 2250 m 2251 ].nodes_best_improved_this_epoch 2252 * 0 2253 ) 2254 layer.dendrite_module.dendrite_values[ 2255 m 2256 ].nodes_improved_any = ( 2257 layer.dendrite_module.dendrite_values[ 2258 m 2259 ].nodes_improved_any 2260 * 0 2261 ) 2262 if GPA.pc.get_perforated_backpropagation(): 2263 self.member_vars["best_mean_score_improved_this_epoch"] = 0 2264 self.member_vars["num_epochs_run"] += 1 2265 self.member_vars["total_epochs_run"] = ( 2266 self.member_vars["num_epochs_run"] + self.member_vars["overwritten_epochs"] 2267 ) 2268 self.saved_time = end
Perform steps for when a new training epoch is about to begin.
Parameters
- internal_call (bool, optional): Whether this is an internal call or manual call
Returns
- None
Notes
If you ever need to call this manually, set internal_call to False.
def
stop_epoch(self, internal_call=False):
2270 def stop_epoch(self, internal_call=False): 2271 """Perform steps when a training epoch has completed. 2272 2273 Parameters 2274 ---------- 2275 internal_call : bool, optional 2276 Whether this is an internal call or manual call 2277 2278 Returns 2279 ------- 2280 None 2281 2282 Notes 2283 ----- 2284 If you ever need to call this manually, set internal_call to False. 2285 2286 """ 2287 end = time.time() 2288 if self.member_vars["manual_train_switch"] and internal_call: 2289 return 2290 2291 if self.member_vars["manual_train_switch"]: 2292 if self.member_vars["mode"] == "p": 2293 self.member_vars["p_train_times"].append(end - self.saved_time) 2294 else: 2295 self.member_vars["n_train_times"].append(end - self.saved_time) 2296 else: 2297 if self.member_vars["mode"] == "p": 2298 self.member_vars["p_epoch_times"].append(end - self.saved_time) 2299 else: 2300 self.member_vars["n_epoch_times"].append(end - self.saved_time) 2301 2302 self.saved_time = end
Perform steps when a training epoch has completed.
Parameters
- internal_call (bool, optional): Whether this is an internal call or manual call
Returns
- None
Notes
If you ever need to call this manually, set internal_call to False.
def
initialize( self, model, doing_pai=True, save_name='PAI', making_graphs=True, maximizing_score=True, num_classes=10000, values_per_train_epoch=-1, values_per_val_epoch=-1, zooming_graph=True):
2304 def initialize( 2305 self, 2306 model, 2307 doing_pai=True, 2308 save_name="PAI", 2309 making_graphs=True, 2310 maximizing_score=True, 2311 num_classes=10000, 2312 values_per_train_epoch=-1, 2313 values_per_val_epoch=-1, 2314 zooming_graph=True, 2315 ): 2316 """Setup the tracker with initial settings. 2317 2318 2319 Parameters 2320 ---------- 2321 model : object 2322 The neural network model. 2323 doing_pai : bool, optional 2324 Whether to add dendrites, by default True. 2325 save_name : str, optional 2326 The name under which to save the model. 2327 making_graphs : bool, optional 2328 Whether to make graphs, by default True. 2329 maximizing_score : bool, optional 2330 Whether to maximize the score, by default True. 2331 num_classes : int, optional 2332 The number of classes in the dataset, unused 2333 values_per_train_epoch : int, optional 2334 The number of values to look back for graphing 2335 during training, by default -1 (all values). 2336 values_per_val_epoch : int, optional 2337 The number of values to look back for graphing 2338 during validation, by default -1 (all values). 2339 zooming_graph : bool, optional 2340 Whether to zoom on graphs, by default True. 2341 2342 """ 2343 model = UPA.convert_network(model) 2344 self.member_vars["doing_pai"] = doing_pai 2345 self.member_vars["maximizing_score"] = maximizing_score 2346 self.save_name = save_name 2347 self.zooming_graph = zooming_graph 2348 self.making_graphs = making_graphs 2349 2350 if not self.loaded: 2351 self.member_vars["running_accuracy"] = (1.0 / num_classes) * 100 2352 2353 self.values_per_train_epoch = values_per_train_epoch 2354 self.values_per_val_epoch = values_per_val_epoch 2355 2356 if GPA.pc.get_testing_dendrite_capacity(): 2357 if not GPA.pc.get_silent(): 2358 print("Running a test of Dendrite Capacity.") 2359 GPA.pc.set_switch_mode(GPA.pc.DOING_SWITCH_EVERY_TIME) 2360 self.member_vars["switch_mode"] = GPA.pc.get_switch_mode() 2361 GPA.pc.set_retain_all_dendrites(True) 2362 GPA.pc.set_max_dendrite_tries(1000) 2363 GPA.pc.set_max_dendrites(1000) 2364 if GPA.pc.get_perforated_backpropagation(): 2365 GPA.pc.set_initial_correlation_batches(1) 2366 else: 2367 if not GPA.pc.get_silent(): 2368 print("Running Dendrite Experiment") 2369 return model
Setup the tracker with initial settings.
Parameters
- model (object): The neural network model.
- doing_pai (bool, optional): Whether to add dendrites, by default True.
- save_name (str, optional): The name under which to save the model.
- making_graphs (bool, optional): Whether to make graphs, by default True.
- maximizing_score (bool, optional): Whether to maximize the score, by default True.
- num_classes (int, optional): The number of classes in the dataset, unused
- values_per_train_epoch (int, optional): The number of values to look back for graphing during training, by default -1 (all values).
- values_per_val_epoch (int, optional): The number of values to look back for graphing during validation, by default -1 (all values).
- zooming_graph (bool, optional): Whether to zoom on graphs, by default True.
def
generate_accuracy_plots(self, ax, save_folder, extra_string):
2371 def generate_accuracy_plots(self, ax, save_folder, extra_string): 2372 """ 2373 Generate plots and csvs for accuracy 2374 2375 Parameters 2376 ---------- 2377 ax : object 2378 The matplotlib axis to plot on. 2379 save_folder : str 2380 The folder to save the plots and csvs in. 2381 extra_string : str 2382 An extra string to append to the filenames. 2383 2384 Returns 2385 ------- 2386 None 2387 2388 """ 2389 2390 # If scores are being saved for epochs that get overwritten, plot them 2391 for list_id in range(len(self.member_vars["overwritten_extras"])): 2392 for extra_id in self.member_vars["overwritten_extras"][list_id]: 2393 ax.plot( 2394 np.arange( 2395 len(self.member_vars["overwritten_extras"][list_id][extra_id]) 2396 ), 2397 self.member_vars["overwritten_extras"][list_id][extra_id], 2398 "r", 2399 ) 2400 ax.plot( 2401 np.arange(len(self.member_vars["overwritten_vals"][list_id])), 2402 self.member_vars["overwritten_vals"][list_id], 2403 "b", 2404 ) 2405 2406 # Determine which accuracy vector to use 2407 if GPA.pc.get_drawing_pai(): 2408 accuracies = self.member_vars["accuracies"] 2409 else: 2410 accuracies = self.member_vars["n_accuracies"] 2411 2412 # Get pointer to additional scores being saved 2413 extra_scores = self.member_vars["extra_scores"] 2414 2415 # Plot the main accuracy scores 2416 ax.plot(np.arange(len(accuracies)), accuracies, label="Validation Scores") 2417 ax.plot( 2418 np.arange(len(self.member_vars["running_accuracies"])), 2419 self.member_vars["running_accuracies"], 2420 label="Validation Running Scores", 2421 ) 2422 2423 # Plot additional scores 2424 for extra_score in extra_scores: 2425 ax.plot( 2426 np.arange(len(extra_scores[extra_score])), 2427 extra_scores[extra_score], 2428 label=extra_score, 2429 ) 2430 2431 plt.title(save_folder + "/" + self.save_name + "Scores") 2432 plt.xlabel("Epochs") 2433 plt.ylabel("Score") 2434 2435 # Add point at epoch last improved and best validation score 2436 if GPA.pc.get_drawing_pai(): 2437 ax.plot( 2438 self.member_vars["epoch_last_improved"], 2439 self.member_vars["global_best_validation_score"], 2440 "bo", 2441 label="Global best (y)", 2442 ) 2443 ax.plot( 2444 self.member_vars["epoch_last_improved"], 2445 accuracies[self.member_vars["epoch_last_improved"]], 2446 "go", 2447 label="Epoch Last Improved", 2448 ) 2449 else: 2450 if self.member_vars["mode"] == "n": 2451 missed_time = ( 2452 self.member_vars["num_epochs_run"] 2453 - self.member_vars["epoch_last_improved"] 2454 ) 2455 ax.plot( 2456 (len(self.member_vars["n_accuracies"]) - 1) - missed_time, 2457 self.member_vars["n_accuracies"][-(missed_time + 1)], 2458 "go", 2459 label="Epoch Last Improved", 2460 ) 2461 2462 # Generate csv file for the values graphed 2463 pd1 = pd.DataFrame( 2464 {"Epochs": np.arange(len(accuracies)), "Validation Scores": accuracies} 2465 ) 2466 pd2 = pd.DataFrame( 2467 { 2468 "Epochs": np.arange(len(self.member_vars["running_accuracies"])), 2469 "Validation Running Scores": self.member_vars["running_accuracies"], 2470 } 2471 ) 2472 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2473 for extra_score in extra_scores: 2474 pd2 = pd.DataFrame( 2475 { 2476 "Epochs": np.arange(len(extra_scores[extra_score])), 2477 extra_score: extra_scores[extra_score], 2478 } 2479 ) 2480 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2481 extra_scores_without_graphing = self.member_vars[ 2482 "extra_scores_without_graphing" 2483 ] 2484 for extra_score in extra_scores_without_graphing: 2485 pd2 = pd.DataFrame( 2486 { 2487 "Epochs": np.arange( 2488 len(extra_scores_without_graphing[extra_score]) 2489 ), 2490 extra_score: extra_scores_without_graphing[extra_score], 2491 } 2492 ) 2493 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2494 pd1.to_csv( 2495 save_folder + "/" + self.save_name + extra_string + "Scores.csv", 2496 index=False, 2497 ) 2498 del pd1, pd2 2499 2500 # Set y min and max to zoom in on important part of axis 2501 if ( 2502 len(self.member_vars["switch_epochs"]) > 0 2503 and self.member_vars["switch_epochs"][0] > 0 2504 and self.zooming_graph 2505 ): 2506 if GPA.pai_tracker.member_vars["maximizing_score"]: 2507 min_val = np.array( 2508 accuracies[0 : self.member_vars["switch_epochs"][0]] 2509 ).mean() 2510 for extra_score in extra_scores: 2511 min_pot = np.array( 2512 extra_scores[extra_score][ 2513 0 : self.member_vars["switch_epochs"][0] 2514 ] 2515 ).mean() 2516 if min_pot < min_val: 2517 min_val = min_pot 2518 ax.set_ylim(ymin=min_val) 2519 else: 2520 max_val = np.array( 2521 accuracies[0 : self.member_vars["switch_epochs"][0]] 2522 ).mean() 2523 for extra_score in extra_scores: 2524 max_pot = np.array( 2525 extra_scores[extra_score][ 2526 0 : self.member_vars["switch_epochs"][0] 2527 ] 2528 ).mean() 2529 if max_pot > max_val: 2530 max_val = max_pot 2531 ax.set_ylim(ymax=max_val) 2532 2533 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2534 2535 # Draw vertical lines for epochs where a dendrite switch occurred 2536 if GPA.pc.get_drawing_pai() and self.member_vars["doing_pai"]: 2537 color = "r" 2538 for switcher in self.member_vars["switch_epochs"]: 2539 plt.axvline(x=switcher, ymin=0, ymax=1, color=color) 2540 if color == "r": 2541 color = "b" 2542 else: 2543 color = "r" 2544 else: 2545 for switcher in self.member_vars["n_switch_epochs"]: 2546 plt.axvline(x=switcher, ymin=0, ymax=1, color="b")
Generate plots and csvs for accuracy
Parameters
- ax (object): The matplotlib axis to plot on.
- save_folder (str): The folder to save the plots and csvs in.
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
generate_time_plots(self, ax, save_folder, extra_string):
2548 def generate_time_plots(self, ax, save_folder, extra_string): 2549 """ 2550 Generate plots and csvs for timing 2551 2552 Parameters 2553 ---------- 2554 ax : object 2555 The matplotlib axis to plot on. 2556 save_folder : str 2557 The folder to save the plots and csvs in. 2558 extra_string : str 2559 An extra string to append to the filenames. 2560 2561 Returns 2562 ------- 2563 None 2564 2565 """ 2566 if self.member_vars["manual_train_switch"]: 2567 ax.plot( 2568 np.arange(len(self.member_vars["n_train_times"])), 2569 self.member_vars["n_train_times"], 2570 label="Normal Epoch Train Times", 2571 ) 2572 ax.plot( 2573 np.arange(len(self.member_vars["p_train_times"])), 2574 self.member_vars["p_train_times"], 2575 label="PAI Epoch Train Times", 2576 ) 2577 ax.plot( 2578 np.arange(len(self.member_vars["n_val_times"])), 2579 self.member_vars["n_val_times"], 2580 label="Normal Epoch Val Times", 2581 ) 2582 ax.plot( 2583 np.arange(len(self.member_vars["p_val_times"])), 2584 self.member_vars["p_val_times"], 2585 label="PAI Epoch Val Times", 2586 ) 2587 2588 plt.title( 2589 save_folder + "/" + self.save_name + "times (by train() and eval())" 2590 ) 2591 plt.xlabel("Iteration") 2592 plt.ylabel("Epoch Time in Seconds ") 2593 ax.set_ylim(ymin=0) 2594 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2595 2596 pd1 = pd.DataFrame( 2597 { 2598 "Epochs": np.arange(len(self.member_vars["n_train_times"])), 2599 "Normal Epoch Train Times": self.member_vars["n_train_times"], 2600 } 2601 ) 2602 pd2 = pd.DataFrame( 2603 { 2604 "Epochs": np.arange(len(self.member_vars["p_train_times"])), 2605 "PAI Epoch Train Times": self.member_vars["p_train_times"], 2606 } 2607 ) 2608 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2609 2610 pd2 = pd.DataFrame( 2611 { 2612 "Epochs": np.arange(len(self.member_vars["n_val_times"])), 2613 "Normal Epoch Val Times": self.member_vars["n_val_times"], 2614 } 2615 ) 2616 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2617 2618 pd2 = pd.DataFrame( 2619 { 2620 "Epochs": np.arange(len(self.member_vars["p_val_times"])), 2621 "PAI Epoch Val Times": self.member_vars["p_val_times"], 2622 } 2623 ) 2624 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2625 2626 pd1.to_csv( 2627 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2628 index=False, 2629 ) 2630 del pd1, pd2 2631 else: 2632 ax.plot( 2633 np.arange(len(self.member_vars["n_epoch_times"])), 2634 self.member_vars["n_epoch_times"], 2635 label="Normal Epoch Times", 2636 ) 2637 ax.plot( 2638 np.arange(len(self.member_vars["p_epoch_times"])), 2639 self.member_vars["p_epoch_times"], 2640 label="PAI Epoch Times", 2641 ) 2642 2643 plt.title( 2644 save_folder + "/" + self.save_name + "times (by train() and eval())" 2645 ) 2646 plt.xlabel("Iteration") 2647 plt.ylabel("Epoch Time in Seconds ") 2648 ax.set_ylim(ymin=0) 2649 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2650 2651 pd1 = pd.DataFrame( 2652 { 2653 "Epochs": np.arange(len(self.member_vars["n_epoch_times"])), 2654 "Normal Epoch Times": self.member_vars["n_epoch_times"], 2655 } 2656 ) 2657 pd2 = pd.DataFrame( 2658 { 2659 "Epochs": np.arange(len(self.member_vars["p_epoch_times"])), 2660 "PAI Epoch Times": self.member_vars["p_epoch_times"], 2661 } 2662 ) 2663 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2664 2665 pd1.to_csv( 2666 save_folder + "/" + self.save_name + extra_string + "Times.csv", 2667 index=False, 2668 ) 2669 del pd1, pd2 2670 2671 if self.values_per_train_epoch != -1 and self.values_per_val_epoch != -1: 2672 ax2 = ax.twinx() # Second axes sharing same x-axis 2673 ax2.set_ylabel("Single Datapoint Time in Seconds") 2674 2675 ax2.plot( 2676 np.arange(len(self.member_vars["n_train_times"])), 2677 np.array(self.member_vars["n_train_times"]) 2678 / self.values_per_train_epoch, 2679 linestyle="dashed", 2680 label="Normal Train Item Times", 2681 ) 2682 ax2.plot( 2683 np.arange(len(self.member_vars["p_train_times"])), 2684 np.array(self.member_vars["p_train_times"]) 2685 / self.values_per_train_epoch, 2686 linestyle="dashed", 2687 label="PAI Train Item Times", 2688 ) 2689 ax2.plot( 2690 np.arange(len(self.member_vars["n_val_times"])), 2691 np.array(self.member_vars["n_val_times"]) / self.values_per_val_epoch, 2692 linestyle="dashed", 2693 label="Normal Val Item Times", 2694 ) 2695 ax2.plot( 2696 np.arange(len(self.member_vars["p_val_times"])), 2697 np.array(self.member_vars["p_val_times"]) / self.values_per_val_epoch, 2698 linestyle="dashed", 2699 label="PAI Val Item Times", 2700 ) 2701 ax2.tick_params(axis="y") 2702 ax2.set_ylim(ymin=0) 2703 ax2.legend(bbox_to_anchor=(1.05, 1), loc="upper left")
Generate plots and csvs for timing
Parameters
- ax (object): The matplotlib axis to plot on.
- save_folder (str): The folder to save the plots and csvs in.
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
generate_learning_rate_plots(self, ax, save_folder, extra_string):
2705 def generate_learning_rate_plots(self, ax, save_folder, extra_string): 2706 """ 2707 Generate plots and csvs for learning rate 2708 2709 Parameters 2710 ---------- 2711 ax : object 2712 The matplotlib axis to plot on. 2713 save_folder : str 2714 The folder to save the plots and csvs in. 2715 extra_string : str 2716 An extra string to append to the filenames. 2717 2718 Returns 2719 ------- 2720 None 2721 2722 """ 2723 ax.plot( 2724 np.arange(len(self.member_vars["training_learning_rates"])), 2725 self.member_vars["training_learning_rates"], 2726 label="learning_rate", 2727 ) 2728 plt.title(save_folder + "/" + self.save_name + "learning_rate") 2729 plt.xlabel("Epochs") 2730 plt.ylabel("learning_rate") 2731 ax.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 2732 2733 pd1 = pd.DataFrame( 2734 { 2735 "Epochs": np.arange(len(self.member_vars["training_learning_rates"])), 2736 "learning_rate": self.member_vars["training_learning_rates"], 2737 } 2738 ) 2739 pd1.to_csv( 2740 save_folder + "/" + self.save_name + extra_string + "learning_rate.csv", 2741 index=False, 2742 ) 2743 del pd1
Generate plots and csvs for learning rate
Parameters
- ax (object): The matplotlib axis to plot on.
- save_folder (str): The folder to save the plots and csvs in.
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
generate_dendrite_learning_plots(self, ax, save_folder, extra_string):
2745 def generate_dendrite_learning_plots(self, ax, save_folder, extra_string): 2746 """ 2747 Generate dendrite score plots for the tracker. 2748 Also saves csv files associated with the plots. 2749 """ 2750 if self.member_vars["doing_pai"]: 2751 pd1 = None 2752 pd2 = None 2753 num_colors = len(self.neuron_module_vector) 2754 2755 if ( 2756 len(self.neuron_module_vector) > 0 2757 and len(self.member_vars["current_scores"][0]) != 0 2758 ): 2759 num_colors *= 2 2760 2761 cm = plt.get_cmap("gist_rainbow") 2762 ax.set_prop_cycle( 2763 "color", [cm(1.0 * i / num_colors) for i in range(num_colors)] 2764 ) 2765 2766 for layer_id in range(len(self.neuron_module_vector)): 2767 ax.plot( 2768 np.arange(len(self.member_vars["best_scores"][layer_id])), 2769 self.member_vars["best_scores"][layer_id], 2770 label=self.neuron_module_vector[layer_id].name, 2771 ) 2772 2773 pd2 = pd.DataFrame( 2774 { 2775 "Epochs": np.arange( 2776 len(self.member_vars["best_scores"][layer_id]) 2777 ), 2778 f"Best ever for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 2779 "best_scores" 2780 ][ 2781 layer_id 2782 ], 2783 } 2784 ) 2785 2786 if pd1 is None: 2787 pd1 = pd2 2788 else: 2789 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2790 2791 if len(self.member_vars["current_scores"][layer_id]) != 0: 2792 ax.plot( 2793 np.arange(len(self.member_vars["current_scores"][layer_id])), 2794 self.member_vars["current_scores"][layer_id], 2795 label=f"Current:{self.neuron_module_vector[layer_id].name}", 2796 ) 2797 2798 pd2 = pd.DataFrame( 2799 { 2800 "Epochs": np.arange( 2801 len(self.member_vars["current_scores"][layer_id]) 2802 ), 2803 f"Best current for all nodes Layer {self.neuron_module_vector[layer_id].name}": self.member_vars[ 2804 "current_scores" 2805 ][ 2806 layer_id 2807 ], 2808 } 2809 ) 2810 pd1 = pd.concat([pd1, pd.DataFrame(pd2)], ignore_index=True) 2811 2812 plt.title(save_folder + "/" + self.save_name + " Best PBScores") 2813 plt.xlabel("Epochs") 2814 plt.ylabel("Best PBScore") 2815 ax.legend( 2816 bbox_to_anchor=(1.05, 1), 2817 loc="upper left", 2818 ncol=max(1, math.ceil(len(self.neuron_module_vector) / 30)), 2819 ) 2820 for switcher in self.member_vars["p_switch_epochs"]: 2821 plt.axvline(x=switcher, ymin=0, ymax=1, color="r") 2822 2823 if self.member_vars["mode"] == "p": 2824 missed_time = ( 2825 self.member_vars["num_epochs_run"] 2826 - self.member_vars["epoch_last_improved"] 2827 ) 2828 plt.axvline( 2829 x=(len(self.member_vars["best_scores"][0]) - (missed_time + 1)), 2830 ymin=0, 2831 ymax=1, 2832 color="g", 2833 ) 2834 2835 # pd1 here will be none if no PB layers are created 2836 if pd1 is not None: 2837 pd1.to_csv( 2838 save_folder 2839 + "/" 2840 + self.save_name 2841 + extra_string 2842 + "Best PBScores.csv", 2843 index=False, 2844 ) 2845 del pd1, pd2
Generate dendrite score plots for the tracker. Also saves csv files associated with the plots.
def
generate_extra_csv_files(self, save_folder, extra_string):
2847 def generate_extra_csv_files(self, save_folder, extra_string): 2848 """ 2849 Generate additional csvs 2850 2851 Parameters 2852 ---------- 2853 save_folder : str 2854 The folder to save the plots and csvs in. 2855 extra_string : str 2856 An extra string to append to the filenames. 2857 2858 Returns 2859 ------- 2860 None 2861 2862 """ 2863 pd1 = pd.DataFrame( 2864 { 2865 "Switch Number": np.arange(len(self.member_vars["switch_epochs"])), 2866 "Switch Epoch": self.member_vars["switch_epochs"], 2867 } 2868 ) 2869 pd1.to_csv( 2870 save_folder + "/" + self.save_name + extra_string + "switch_epochs.csv", 2871 index=False, 2872 ) 2873 del pd1 2874 2875 pd1 = pd.DataFrame( 2876 { 2877 "Switch Number": np.arange(len(self.member_vars["param_counts"])), 2878 "Param Count": self.member_vars["param_counts"], 2879 } 2880 ) 2881 pd1.to_csv( 2882 save_folder + "/" + self.save_name + extra_string + "param_counts.csv", 2883 index=False, 2884 ) 2885 del pd1 2886 2887 """ 2888 Create best_arch_scores.csv file 2889 When working with dendrites there is a tradeoff between additional param count and score improvement. 2890 This file will help track that tradeoff by recording the best scores for all extra_scores 2891 and extra_scores_without_graphing for each architecture version. 2892 The scores recorded here are from the epoch when the best validation score was found 2893 within each switch_epoch boundary. 2894 """ 2895 switch_counts = len(self.member_vars["switch_epochs"]) 2896 best_valid = [] 2897 associated_params = [] 2898 2899 # Initialize dictionaries to store best scores for each extra score type 2900 best_extra_scores = {} 2901 for score_name in self.member_vars["extra_scores"]: 2902 best_extra_scores[score_name] = [] 2903 for score_name in self.member_vars["extra_scores_without_graphing"]: 2904 best_extra_scores[score_name] = [] 2905 2906 for switch in range(0, switch_counts, 2): 2907 start_index = 0 2908 if switch != 0: 2909 start_index = self.member_vars["switch_epochs"][switch - 1] + 1 2910 end_index = self.member_vars["switch_epochs"][switch] + 1 2911 2912 if GPA.pai_tracker.member_vars["maximizing_score"]: 2913 best_valid_index = start_index + np.argmax( 2914 self.member_vars["accuracies"][start_index:end_index] 2915 ) 2916 else: 2917 best_valid_index = start_index + np.argmin( 2918 self.member_vars["accuracies"][start_index:end_index] 2919 ) 2920 2921 best_valid_score = self.member_vars["accuracies"][best_valid_index] 2922 best_valid.append(best_valid_score) 2923 2924 # Get corresponding scores from all extra_scores 2925 for score_name in self.member_vars["extra_scores"]: 2926 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 2927 best_extra_scores[score_name].append( 2928 self.member_vars["extra_scores"][score_name][best_valid_index] 2929 ) 2930 else: 2931 best_extra_scores[score_name].append(None) 2932 2933 # Get corresponding scores from all extra_scores_without_graphing 2934 for score_name in self.member_vars["extra_scores_without_graphing"]: 2935 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 2936 best_extra_scores[score_name].append( 2937 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 2938 ) 2939 else: 2940 best_extra_scores[score_name].append(None) 2941 2942 if self.member_vars["doing_pai"]: 2943 associated_params.append(self.member_vars["param_counts"][switch]) 2944 else: 2945 associated_params.append(self.member_vars["param_counts"][-1]) 2946 2947 # If in neuron training mode but not the very first epoch 2948 if self.member_vars["mode"] == "n" and ( 2949 (len(self.member_vars["switch_epochs"]) == 0) 2950 or ( 2951 self.member_vars["switch_epochs"][-1] + 1 2952 != len(self.member_vars["accuracies"]) 2953 ) 2954 ): 2955 start_index = 0 2956 if len(self.member_vars["switch_epochs"]) != 0: 2957 start_index = self.member_vars["switch_epochs"][-1] + 1 2958 2959 if GPA.pai_tracker.member_vars["maximizing_score"]: 2960 best_valid_index = start_index + np.argmax( 2961 self.member_vars["accuracies"][start_index:] 2962 ) 2963 else: 2964 best_valid_index = start_index + np.argmin( 2965 self.member_vars["accuracies"][start_index:] 2966 ) 2967 2968 best_valid_score = self.member_vars["accuracies"][best_valid_index] 2969 best_valid.append(best_valid_score) 2970 2971 # Get corresponding scores from all extra_scores 2972 for score_name in self.member_vars["extra_scores"]: 2973 if best_valid_index < len(self.member_vars["extra_scores"][score_name]): 2974 best_extra_scores[score_name].append( 2975 self.member_vars["extra_scores"][score_name][best_valid_index] 2976 ) 2977 else: 2978 best_extra_scores[score_name].append(None) 2979 2980 # Get corresponding scores from all extra_scores_without_graphing 2981 for score_name in self.member_vars["extra_scores_without_graphing"]: 2982 if best_valid_index < len(self.member_vars["extra_scores_without_graphing"][score_name]): 2983 best_extra_scores[score_name].append( 2984 self.member_vars["extra_scores_without_graphing"][score_name][best_valid_index] 2985 ) 2986 else: 2987 best_extra_scores[score_name].append(None) 2988 2989 associated_params.append(self.member_vars["param_counts"][-1]) 2990 2991 # Build dataframe with all columns 2992 csv_data = { 2993 "Param Counts": associated_params, 2994 "Max Valid Scores": best_valid, 2995 } 2996 2997 # Add columns for each extra score 2998 for score_name in best_extra_scores: 2999 csv_data[score_name] = best_extra_scores[score_name] 3000 3001 pd1 = pd.DataFrame(csv_data) 3002 pd1.to_csv( 3003 save_folder + "/" + self.save_name + extra_string + "_best_arch_scores.csv", 3004 index=False, 3005 ) 3006 del pd1
Generate additional csvs
Parameters
- save_folder (str): The folder to save the plots and csvs in.
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
save_graphs(self, extra_string=''):
3008 def save_graphs(self, extra_string=""): 3009 """ 3010 Save graphs and csvs for all the values the tracker records 3011 3012 Parameters 3013 ---------- 3014 extra_string : str 3015 An extra string to append to the filenames. 3016 3017 Returns 3018 ------- 3019 None 3020 3021 """ 3022 # If running DDP only save with rank 0 3023 if "RANK" in os.environ: 3024 if int(os.environ["RANK"]) != 0: 3025 return 3026 if not self.making_graphs: 3027 return 3028 3029 save_folder = "./" + self.save_name + "/" 3030 3031 plt.ioff() 3032 fig = plt.figure(figsize=(28, 14)) 3033 3034 # Plot with accuracy scores 3035 ax = plt.subplot(221) 3036 self.generate_accuracy_plots(ax, save_folder, extra_string) 3037 3038 # Plot dendrite learning scores 3039 ax = plt.subplot(222) 3040 self.generate_dendrite_learning_plots(ax, save_folder, extra_string) 3041 3042 if GPA.pc.get_drawing_extra_graphs(): 3043 # Plot learning rates for each training epoch 3044 ax = plt.subplot(223) 3045 self.generate_learning_rate_plots(ax, save_folder, extra_string) 3046 3047 # Plot the times for each training epoch 3048 ax = plt.subplot(224) 3049 self.generate_time_plots(ax, save_folder, extra_string) 3050 3051 # Generate extra CSV files 3052 self.generate_extra_csv_files(save_folder, extra_string) 3053 3054 fig.tight_layout() 3055 plt.savefig(save_folder + "/" + self.save_name + extra_string + ".png") 3056 plt.close("all")
Save graphs and csvs for all the values the tracker records
Parameters
- extra_string (str): An extra string to append to the filenames.
Returns
- None
def
add_loss(self, loss):
3058 def add_loss(self, loss): 3059 """Add loss to tracking vectors. 3060 3061 Parameters 3062 ---------- 3063 loss : float or int 3064 The loss value to add. 3065 3066 Returns 3067 ------- 3068 None 3069 3070 """ 3071 if not isinstance(loss, (float, int)): 3072 loss = loss.item() 3073 self.member_vars["training_loss"].append(loss)
Add loss to tracking vectors.
Parameters
- loss (float or int): The loss value to add.
Returns
- None
def
add_learning_rate(self, learning_rate):
3075 def add_learning_rate(self, learning_rate): 3076 """Add learning rate to tracking vectors. 3077 3078 Parameters 3079 ---------- 3080 learning_rate : float or int 3081 The learning rate value to add. 3082 3083 Returns 3084 ------- 3085 None 3086 3087 """ 3088 if not isinstance(learning_rate, (float, int)): 3089 learning_rate = learning_rate.item() 3090 self.member_vars["training_learning_rates"].append(learning_rate)
Add learning rate to tracking vectors.
Parameters
- learning_rate (float or int): The learning rate value to add.
Returns
- None
def
add_extra_score(self, score, extra_score_name):
3092 def add_extra_score(self, score, extra_score_name): 3093 """Add extra score to tracking vectors. 3094 3095 Parameters 3096 ---------- 3097 score : float or int 3098 The score value to add. 3099 3100 extra_score_name : str 3101 The name of the extra score. 3102 3103 Returns 3104 ------- 3105 None 3106 3107 """ 3108 if not isinstance(score, (float, int)): 3109 try: 3110 score = score.item() 3111 except: 3112 print( 3113 "Scores added for Perforated Backpropagation should be " 3114 "float, int, or tensor, yours is a:" 3115 ) 3116 print(type(score)) 3117 pdb.set_trace() 3118 3119 if GPA.pc.get_verbose(): 3120 print(f"Adding extra score {extra_score_name} of {float(score)}") 3121 3122 if extra_score_name not in self.member_vars["extra_scores"]: 3123 self.member_vars["extra_scores"][extra_score_name] = [] 3124 self.member_vars["extra_scores"][extra_score_name].append(score) 3125 3126 if self.member_vars["mode"] == "n": 3127 if extra_score_name not in self.member_vars["n_extra_scores"]: 3128 self.member_vars["n_extra_scores"][extra_score_name] = [] 3129 self.member_vars["n_extra_scores"][extra_score_name].append(score)
Add extra score to tracking vectors.
Parameters
- score (float or int): The score value to add.
- extra_score_name (str): The name of the extra score.
Returns
- None
def
add_extra_score_without_graphing(self, score, extra_score_name):
3131 def add_extra_score_without_graphing(self, score, extra_score_name): 3132 """Add extra score without graphing to tracking vectors. 3133 3134 Parameters 3135 ---------- 3136 score : float or int 3137 The score value to add. 3138 3139 extra_score_name : str 3140 The name of the extra score. 3141 3142 Returns 3143 ------- 3144 None 3145 3146 """ 3147 if not isinstance(score, (float, int)): 3148 try: 3149 score = score.item() 3150 except: 3151 print( 3152 "Scores added for Perforated Backpropagation should be " 3153 "float, int, or tensor, yours is a:" 3154 ) 3155 print(type(score)) 3156 print("in add_extra_score_without_graphing") 3157 pdb.set_trace() 3158 3159 if GPA.pc.get_verbose(): 3160 print(f"Adding extra score {extra_score_name} of {float(score)}") 3161 3162 if extra_score_name not in self.member_vars["extra_scores_without_graphing"]: 3163 self.member_vars["extra_scores_without_graphing"][extra_score_name] = [] 3164 self.member_vars["extra_scores_without_graphing"][extra_score_name].append( 3165 score 3166 )
Add extra score without graphing to tracking vectors.
Parameters
- score (float or int): The score value to add.
- extra_score_name (str): The name of the extra score.
Returns
- None
def
add_test_score(self, score, extra_score_name):
3168 def add_test_score(self, score, extra_score_name): 3169 """Add test score to tracking vectors. 3170 3171 Parameters 3172 ---------- 3173 score : float or int 3174 The score value to add. 3175 3176 extra_score_name : str 3177 The name of the extra score. 3178 3179 Returns 3180 ------- 3181 None 3182 3183 Notes 3184 ----- 3185 This function is a wrapper around `add_extra_score` that separates 3186 test score for adding to best_arch_scores.csv. 3187 3188 """ 3189 self.add_extra_score(score, extra_score_name) 3190 3191 if not isinstance(score, (float, int)): 3192 try: 3193 score = score.item() 3194 except: 3195 print( 3196 "Scores added for Perforated Backpropagation should be " 3197 "float, int, or tensor, yours is a:" 3198 ) 3199 print(type(score)) 3200 print("in add_test_score") 3201 pdb.set_trace() 3202 3203 if GPA.pc.get_verbose(): 3204 print(f"Adding test score {extra_score_name} of {float(score)}") 3205 self.member_vars["test_scores"].append(score)
Add test score to tracking vectors.
Parameters
- score (float or int): The score value to add.
- extra_score_name (str): The name of the extra score.
Returns
- None
Notes
This function is a wrapper around add_extra_score that separates
test score for adding to best_arch_scores.csv.
def
add_validation_score(self, accuracy, net, force_switch=False):
3207 def add_validation_score(self, accuracy, net, force_switch=False): 3208 """Function to add the validation score. 3209 3210 This is complex because it determines neuron and dendrite switching. 3211 3212 Parameters 3213 ---------- 3214 accuracy : float or int 3215 The accuracy or loss value to add. 3216 net : object 3217 The neural network model. 3218 force_switch : bool, optional 3219 Whether to force a switch, by default False. 3220 3221 Returns 3222 ------- 3223 net : object 3224 The potentially modified neural network model. 3225 training_complete : bool 3226 Whether training is complete. 3227 restructured : bool 3228 Whether the model has been restructured. 3229 3230 Notes 3231 ----- 3232 WARNING: Do not call self anywhere in this function. When systems 3233 get loaded the actual tracker you are working with can change. 3234 """ 3235 3236 if not GPA.pc.get_silent(): 3237 print(f"Adding validation score {accuracy:.8f}") 3238 3239 update_learning_rate() 3240 update_param_count(net) 3241 3242 accuracy = check_input_problems(net, accuracy) 3243 3244 if len(GPA.pai_tracker.member_vars["switch_epochs"]) == 0: 3245 epochs_since_cycle_switch = GPA.pai_tracker.member_vars["num_epochs_run"] 3246 else: 3247 epochs_since_cycle_switch = ( 3248 GPA.pai_tracker.member_vars["num_epochs_run"] 3249 - GPA.pai_tracker.member_vars["switch_epochs"][-1] 3250 ) 3251 3252 update_running_accuracy(accuracy, epochs_since_cycle_switch) 3253 if GPA.pc.get_perforated_backpropagation(): 3254 TPB.update_pb_scores(self) 3255 3256 GPA.pai_tracker.stop_epoch(internal_call=True) 3257 3258 # If it is neuron training mode 3259 if ( 3260 GPA.pai_tracker.member_vars["mode"] == "n" 3261 or GPA.pc.get_learn_dendrites_live() 3262 ): 3263 check_new_best(net, accuracy, epochs_since_cycle_switch) 3264 elif GPA.pc.get_perforated_backpropagation(): 3265 TPB.check_best_pai_score_improvement() 3266 3267 # Save the latest model 3268 if GPA.pc.get_test_saves(): 3269 UPA.save_system(net, GPA.pc.get_save_name(), "latest") 3270 if GPA.pc.get_pai_saves(): 3271 UPA.pai_save_system(net, GPA.pc.get_save_name(), "latest") 3272 3273 restructuring_status_value = NO_MODEL_UPDATE 3274 # If it is time to switch based on scores and counter or a manual switch 3275 if GPA.pai_tracker.switch_time() or force_switch: 3276 # If testing dendrite capacity switch after enough dendrites added 3277 if ( 3278 (GPA.pai_tracker.member_vars["mode"] == "n") 3279 and (GPA.pai_tracker.member_vars["num_dendrites_added"] > 2) 3280 and GPA.pc.get_testing_dendrite_capacity() 3281 ): 3282 GPA.pai_tracker.save_graphs() 3283 print( 3284 "Successfully added 3 dendrites with " 3285 "GPA.pc.set_testing_dendrite_capacity(True) (default). " 3286 "You may now set that to False and run a real experiment." 3287 ) 3288 return net, False, True 3289 3290 # If doing neuron training but this dendrite count didn't improve 3291 if ( 3292 (GPA.pai_tracker.member_vars["mode"] == "n") 3293 or GPA.pc.get_learn_dendrites_live() 3294 ) and (GPA.pai_tracker.member_vars["current_n_set_global_best"] is False): 3295 new_restructuring_status_value, net = process_no_improvement(net) 3296 # if this was the final try return that training is complete 3297 if new_restructuring_status_value == TRAINING_COMPLETE: 3298 return net, True, True 3299 else: 3300 restructuring_status_value = update_restructuring_status( 3301 restructuring_status_value, new_restructuring_status_value 3302 ) 3303 # Else if did improve, do a normal switch process 3304 else: 3305 if GPA.pc.get_verbose(): 3306 print( 3307 f"Calling switch_mode with " 3308 f'{GPA.pai_tracker.member_vars["current_n_set_global_best"]}, ' 3309 f'{GPA.pai_tracker.member_vars["current_n_learning_rate_initial_skip_steps"]}, ' 3310 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_steps"]}, ' 3311 f'{GPA.pai_tracker.member_vars["last_max_learning_rate_value"]},' 3312 f'{GPA.pc.get_max_dendrites()},' 3313 f'{GPA.pai_tracker.member_vars["num_dendrites_added"]},' 3314 f'{GPA.pai_tracker.member_vars["num_dendrite_tries"]},' 3315 ) 3316 import pdb; pdb.set_trace 3317 # If the max number of dendrites has been hit or not doing pai and adding dendtites 3318 # then return rather than adding more 3319 if ( 3320 (GPA.pai_tracker.member_vars["mode"] == "n") 3321 and ( 3322 GPA.pc.get_max_dendrites() 3323 == GPA.pai_tracker.member_vars["num_dendrites_added"] 3324 ) 3325 ) or (GPA.pai_tracker.member_vars["doing_pai"] is False): 3326 if GPA.pc.get_verbose(): 3327 print( 3328 "Max dendrites reached or not doing PAI, finishing training" 3329 ) 3330 net = process_final_network(net) 3331 # Increment integrated if we have dendrites (means they're integrated) 3332 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3333 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3334 if not GPA.pc.get_silent(): 3335 print(f"Final dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3336 return net, True, True 3337 3338 # Otherwise if its neuron training mode reset the counter of failed dendrites 3339 # Check if we should increment integrated count BEFORE change_learning_modes loads old state 3340 should_increment_integrated = False 3341 if GPA.pai_tracker.member_vars["mode"] == "n": 3342 GPA.pai_tracker.member_vars["num_dendrite_tries"] = 0 3343 if GPA.pc.get_verbose(): 3344 print( 3345 "Adding new dendrites without resetting which means " 3346 "the last ones improved. Resetting num_dendrite_tries" 3347 ) 3348 # Remember to increment after change_learning_modes (which loads old tracker state) 3349 if GPA.pai_tracker.member_vars["num_dendrites_added"] > 0: 3350 should_increment_integrated = True 3351 3352 GPA.pai_tracker.save_graphs( 3353 f'_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}' 3354 ) 3355 3356 if GPA.pc.get_test_saves(): 3357 UPA.save_system( 3358 net, 3359 GPA.pc.get_save_name(), 3360 f'beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3361 ) 3362 # Copy current best model from this set of dendrites 3363 # If running DDP only copy with rank 0 3364 if "RANK" not in os.environ or int(os.environ["RANK"]) == 0: 3365 shutil.copyfile( 3366 f"{GPA.pc.get_save_name()}/best_model.pt", 3367 f'{GPA.pc.get_save_name()}/best_model_beforeSwitch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}.pt', 3368 ) 3369 3370 net = UPA.change_learning_modes( 3371 net, 3372 GPA.pc.get_save_name(), 3373 "best_model", 3374 GPA.pai_tracker.member_vars["doing_pai"], 3375 ) 3376 restructuring_status_value = NETWORK_RESTRUCTURED 3377 3378 # Now increment after change_learning_modes has loaded the best model 3379 # This ensures the increment persists and doesn't get overwritten 3380 if should_increment_integrated: 3381 GPA.pai_tracker.member_vars["num_dendrites_integrated"] += 1 3382 if not GPA.pc.get_silent(): 3383 print(f"Dendrites successfully integrated! Total integrated: {GPA.pai_tracker.member_vars['num_dendrites_integrated']}") 3384 3385 # If restructured is true, clear scheduler/optimizer before saving 3386 if restructuring_status_value != NETWORK_RESTRUCTURED: 3387 print( 3388 "Restructured should always be triggered here, let us know if you encounter this situation" 3389 ) 3390 pdb.set_trace() 3391 3392 # Since there is a restructuring optimizer and scheduler must be reinitialized after return 3393 GPA.pai_tracker.clear_optimizer_and_scheduler() 3394 3395 # Save the model from after the switch 3396 UPA.save_system( 3397 net, 3398 GPA.pc.get_save_name(), 3399 f'switch_{len(GPA.pai_tracker.member_vars["switch_epochs"])}', 3400 ) 3401 3402 # If not time to switch and you have a scheduler, perform the update step 3403 elif GPA.pai_tracker.member_vars["scheduler"] is not None: 3404 new_restructuring_status_value, net = process_scheduler_update( 3405 net, accuracy, epochs_since_cycle_switch 3406 ) 3407 restructuring_status_value = update_restructuring_status( 3408 restructuring_status_value, new_restructuring_status_value 3409 ) 3410 3411 GPA.pai_tracker.start_epoch(internal_call=True) 3412 GPA.pai_tracker.save_graphs() 3413 3414 if restructuring_status_value == NETWORK_RESTRUCTURED: 3415 GPA.pai_tracker.member_vars["epoch_last_improved"] = ( 3416 GPA.pai_tracker.member_vars["num_epochs_run"] 3417 ) 3418 if GPA.pc.get_verbose(): 3419 print( 3420 f"Setting epoch last improved to " 3421 f'{GPA.pai_tracker.member_vars["epoch_last_improved"]}' 3422 ) 3423 3424 now = datetime.now() 3425 dt_string = now.strftime("_%d.%m.%Y.%H.%M.%S") 3426 3427 if GPA.pc.get_verbose(): 3428 print("Not saving restructure right now") 3429 3430 """ 3431 This block of code helped with a save issue with safetensors and huggingface, but it breaks DDP. 3432 Temporarily removing it to avoid DDP issues, but if you encounter save issues try adding it back in. 3433 for param in net.parameters(): 3434 param.data = param.data.contiguous() 3435 """ 3436 if GPA.pc.get_verbose(): 3437 print( 3438 f"Completed adding score. Restructured is {restructuring_status_value}, " 3439 f"\ncurrent switch list is:" 3440 ) 3441 print(GPA.pai_tracker.member_vars["switch_epochs"]) 3442 3443 # Always False for training complete if nothing triggered that training is over 3444 return net, restructuring_status_value, False
Function to add the validation score.
This is complex because it determines neuron and dendrite switching.
Parameters
- accuracy (float or int): The accuracy or loss value to add.
- net (object): The neural network model.
- force_switch (bool, optional): Whether to force a switch, by default False.
Returns
- net (object): The potentially modified neural network model.
- training_complete (bool): Whether training is complete.
- restructured (bool): Whether the model has been restructured.
Notes
WARNING: Do not call self anywhere in this function. When systems get loaded the actual tracker you are working with can change.
def
clear_all_processors(self):
3446 def clear_all_processors(self): 3447 """Clear all processors from modules.""" 3448 for module in self.neuron_module_vector: 3449 module.clear_processors()
Clear all processors from modules.
def
create_new_dendrite_module(self):
3451 def create_new_dendrite_module(self): 3452 """Add dendrite module to all neuron modules.""" 3453 for module in self.neuron_module_vector: 3454 module.create_new_dendrite_module()
Add dendrite module to all neuron modules.
def
apply_pb_grads(self):
3456 def apply_pb_grads(self): 3457 """Apply perforated backpropagation gradients to all modules.""" 3458 if self.member_vars["mode"] == "p": 3459 for module in self.neuron_module_vector: 3460 module.apply_pb_grads()
Apply perforated backpropagation gradients to all modules.
def
apply_pb_zero(self):
3462 def apply_pb_zero(self): 3463 """Apply perforated backpropagation zero gradients to all modules.""" 3464 if self.member_vars["mode"] == "p": 3465 for module in self.neuron_module_vector: 3466 module.apply_pb_zero()
Apply perforated backpropagation zero gradients to all modules.