terminal output improvements + flush=True for print()

.gitignore

@@ -126,7 +126,6 @@ dmypy.json
 .pyre/
 
 # BBGradebookOrganiser
-TODO
 BB_gradebooks/
 BB_submissions/
 csv-inspect/
@@ -138,3 +137,10 @@ csv-inspect/
 
 mkdocs.yml
 /site
+
+# vangef
+
+__*.py
+venv*
+.TODO
+.NOTES

organise_gradebook.py

@@ -1,14 +1,15 @@
 import os, sys
 from utils.organiser import organise_gradebook, check_submissions_dir_for_compressed
 
+
 def main():
     gradebook_name = ' '.join(sys.argv[1:]) if len(sys.argv) > 1 else exit(f'\nNo gradebook name given. Provide the name as an argument.\n\nUsage: python {sys.argv[0]} [gradebook dir name]\n')
     gradebook_dir = os.path.join('BB_gradebooks', gradebook_name)  # gradebook from Blackboard with all submissions
     submissions_dir = os.path.join('BB_submissions', gradebook_name)  # target dir for extracted submissions
 
     abs_path = os.getcwd()  # absolute path of main/this script
-    print(f'\nGradebook directory to organise: {os.path.join(abs_path, gradebook_dir)}')
+    print(f'\nGradebook directory to organise:\n{os.path.join(abs_path, gradebook_dir)}', flush=True)
-    
+        
     organise_gradebook(gradebook_dir, submissions_dir)
     check_submissions_dir_for_compressed(submissions_dir) 
 

requirements.txt

@@ -0,0 +1,5 @@
+# for organise gradebook script
+py7zr
+rarfile
+# for inspect gradebook/submissions scripts
+pandas

utils/extractor.py

@@ -11,9 +11,9 @@ def mark_file_as_BAD(file: str, bad_exception: Exception) -> None:
         os.makedirs(bad_dir, exist_ok=True)
         bad_file_path = os.path.join(bad_dir, filename)
         shutil.move(file, bad_file_path)
-        print(f'\n[Warning] Found BAD compressed file: {filename}\nMoved to: {bad_file_path}\nError message: {bad_exception}')
+        print(f'\n[Warning] Found BAD compressed file: {filename}\nMoved to: {bad_file_path}\nError message: {bad_exception}\n', flush=True)
     except Exception as e: 
-        print(f'\n[Error] {e}')
+        print(f'\n[ERROR] {e}\n', flush=True)
 
 def extract_zip(zip_file: str, target_dir: str) -> None | Exception:
     try:
@@ -24,7 +24,7 @@ def extract_zip(zip_file: str, target_dir: str) -> None | Exception:
     except zipfile.BadZipfile as e:
         mark_file_as_BAD(zip_file, e)
     except Exception as e:
-        print(f'\n[ERROR] Something went wrong while extracting the contents of a submitted zip file. Check the error message, get student id and download / organise manually\nError message: {e}')
+        print(f'\n[ERROR] Something went wrong while extracting the contents of a submitted zip file. Check the error message, get student id and download / organise manually\n\nError message: {e}\n', flush=True)
         return e
 
 def extract_rar(rar_file: str, target_dir: str) -> None:
@@ -45,7 +45,7 @@ def extract_rar(rar_file: str, target_dir: str) -> None:
     except rarfile.NotRarFile as e:
         mark_file_as_BAD(rar_file, e)
     except rarfile.RarCannotExec as e:
-        print('\n[Error] Missing unrar tool\nfor Windows: make sure file UnRAR.exe exists in directory \'utils\'\nfor Linux/Mac: need to install unrar (check README)')
+        print('\n[ERROR] Missing unrar tool\nfor Windows: make sure file UnRAR.exe exists in directory \'utils\'\nfor Linux/Mac: need to install unrar (check README)\n', flush=True)
         exit()
 
 def extract_7z(seven_zip_file: str, target_dir: str) -> None:
@@ -73,4 +73,4 @@ def extract_file_to_dir(file_path: str, student_dir: str) -> None | Exception:
     elif file_path.lower().endswith('.7z'):
         extract_7z(file_path, student_dir) 
     else:
-        print(f"\n[Error] unknown file type: {file_path}")
+        print(f"\n[ERROR] unknown file type: {file_path}\n", flush=True)

utils/inspector.py

@@ -11,18 +11,18 @@ from utils.settings import CSV_DIR
 def load_excluded_filenames(submissions_dir_name: str) -> list[str]:  # helper function for hashing all files
     csv_file_path = os.path.join(CSV_DIR, f'{submissions_dir_name}_excluded.csv')
     if not os.path.exists(csv_file_path):  # if csv file with excluded file names for submission does not exist
-        print(f'[WARNING] Cannot find CSV file with list of excluded file names: {csv_file_path}\n[INFO] All files will be hashed & inspected')
+        print(f'[WARNING] Cannot find CSV file with list of excluded file names: {csv_file_path}\n[INFO] All files will be hashed & inspected', flush=True)
         return []  # return empty list to continue without any excluded file names
     else:  # if csv file with excluded file names for submission exists
         try:            
             df = pd.read_csv(csv_file_path)
             filename_list = df['exclude_filename'].tolist()  # get the values of the 'filename' column as a list
             filename_list = [ f.lower() for f in filename_list ]  # convert to lowercase for comparison with submission files
-            print(f'[INFO] Using CSV file with list of excluded file names: {csv_file_path}')
+            print(f'[INFO] Using CSV file with list of excluded file names: {csv_file_path}', flush=True)
             return filename_list
         except Exception as e:  # any exception, print error and return empty list to continue without any excluded file names
-            print(f'[WARNING] Unable to load / read CSV file with list of excluded file names: {csv_file_path}\n[INFO] All files will be hashed & inspected')
+            print(f'[WARNING] Unable to load / read CSV file with list of excluded file names: {csv_file_path}\n[INFO] All files will be hashed & inspected', flush=True)
-            print(f'[INFO] Error message: {e}')
+            print(f'[INFO] Error message: {e}', flush=True)
             return []
 
 
@@ -61,7 +61,7 @@ def generate_hashes_gradebook(gradebook_dir_path: str) -> str:  # main function
         writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
         writer.writeheader()
         writer.writerows(dicts_with_hashes_list)
-    print(f'[INFO] Created CSV file with all files & hashes in gradebook: {gradebook_dir_name}\nCSV file: {csv_file_path}')
+    print(f'[INFO] Created CSV file with all files & hashes in gradebook: {gradebook_dir_name}\nCSV file: {csv_file_path}', flush=True)
     return csv_file_path
 
 
@@ -95,7 +95,7 @@ def generate_hashes_submissions(submissions_dir_path: str) -> str:  # main funct
         writer.writeheader()
         for student_dict in dicts_with_hashes_list:
             writer.writerows(student_dict)
-    print(f'[INFO] Created CSV file with all files & hashes for submissions in: {submissions_dir_name}\nCSV file: {csv_file_path}')
+    print(f'[INFO] Created CSV file with all files & hashes for submissions in: {submissions_dir_name}\nCSV file: {csv_file_path}', flush=True)
     return csv_file_path
    
 
@@ -119,7 +119,7 @@ def generate_duplicate_hashes_generic(hashes_csv_file_path: str, drop_columns: l
     csv_out = hashes_csv_file_path.rsplit('_', 1)[0].replace('file_hashes', 'duplicate_') + datetime.now().strftime("%Y%m%d-%H%M%S") + '.csv'
     try:
         df_duplicate.to_csv(csv_out, index=False)
-        print(f'[INFO] Created CSV file with duplicate hashes in {gradebook_or_submissions_str}: {assignment_name}\nCSV file: {csv_out}')
+        print(f'[INFO] Created CSV file with duplicate hashes in {gradebook_or_submissions_str}: {assignment_name}\nCSV file: {csv_out}', flush=True)
     except Exception as e:
         exit(f'[ERROR] Something went wrong while trying to save csv file with duplicate hashes\nError message: {e}')
 

utils/organiser.py

@@ -2,7 +2,6 @@ import os, shutil, re
 from utils.extractor import extract_file_to_dir
 from utils.settings import BAD_DIR_NAME
 
-
 def validate_gradebook_dir_name(src_dir: str) -> None:
     if not os.path.isdir(src_dir):  # check if it exists and is a directory
         print(f"\n[Error] Incorrect directory: {src_dir}\n[Info] Make sure the directory exists in 'BB_gradebooks'")
@@ -53,7 +52,7 @@ def get_gradebook_stats(src_dir: str) -> dict[str, int]:
     tracked_files_list = [ f'{files_counter[ext]} {ext}' for ext in tracked_file_extensions ] 
     tracked_msg = f"{', '.join(str(f) for f in tracked_files_list)}"
     msg = f'\n[Stats] Gradebook contains {files_counter["all"]} file(s){dirs_msg}\n[Stats] Tracking {len(tracked_file_extensions)} file extension(s), files found: {tracked_msg}\n[Stats] Files with untracked extension: {files_counter["untracked"]}'
-    print(msg)
+    print(msg, flush=True)
     return files_counter
 
 
@@ -88,10 +87,11 @@ def organise_gradebook(src_dir: str, dest_dir: str) -> None:
     """
     validate_gradebook_dir_name(src_dir)  # check if dir exists, and has files in it - exits if not
     os.makedirs(dest_dir, exist_ok=True)  # create the destination directory if it doesn't exist
-    print('\nGetting gradebook stats...')
+    print('\nGetting gradebook stats...', flush=True)
     files_counter = get_gradebook_stats(src_dir)  # print stats about the files in gradebook and get files_counter dict to use later
     students_numbers: list[str] = []  # list to add and count unique student numbers from all files in gradebook 
-    print('\nStart organising...\n')
+    print('\nStart organising... (this may take a while depending on the number of submissions)\n', flush=True)
+
     for file_name in os.listdir(src_dir):  # iterate through all files in the directory
         if BAD_DIR_NAME not in file_name:  # ignore dir BAD_DIR_NAME (created after first run if corrupt compressed files found)
             student_no = file_name.split('_attempt_')[0].split('_')[-1]  # get student number from file name !! pattern might need adjusting if file name format from blackboard changes !!
@@ -99,14 +99,14 @@ def organise_gradebook(src_dir: str, dest_dir: str) -> None:
             organise_file_per_student(src_dir, dest_dir, file_name, student_no)
     
     abs_path = os.getcwd()  # absolute path of main script
-    print(f'[Info] Submissions organised into directory: {os.path.join(abs_path, dest_dir)}')
+    print(f'[Info] Submissions organised into directory: {os.path.join(abs_path, dest_dir)}', flush=True)
-    print(f'[Info] Unique student numbers in gradebook files: {len(set(students_numbers))}')
+    print(f'[Info] Unique student numbers in gradebook files: {len(set(students_numbers))}', flush=True)
     if files_counter['.txt'] == 0:
-        print(f'[Info] No submission text files found, file with comments not created')
+        print(f'[Info] No submission text files found, file with comments not created', flush=True)
     else:
-        print(f'[Info] Comments in file: {dest_dir}_comments.txt')
+        print(f'[Info] Comments in file: {dest_dir}_comments.txt', flush=True)
     
-    print(f'[Note] Compressed files (.zip, .rar, .7z) are automatically deleted from the gradebook directory after successful extraction')
+    print(f'[Note] Compressed files (.zip, .rar, .7z) are automatically deleted from the gradebook directory after successful extraction', flush=True)
 
     
 def check_submissions_dir_for_compressed(submissions_dir: str) -> None: