SML Docs: Updated Bulk Sound Extract (cleaned up)

modules/ROOT/attachments/Development/SatisfactoryAudioRenamer/convert.py

@@ -1,31 +1,215 @@
 import os
+import subprocess
 from pathlib import Path
+import shutil
+import logging
 
-def convert(filename):
+# Full path to vgmstream-cli.exe, usually in your FModel's Output Directory
+# Example Path
+VGMSTREAM = Path(r"C:/FModel/Output/.data/vgmstream-cli.exe")
 
-    my_file = open("./txtp/" + filename, "r")
-    data = my_file.read()
+# Logs
+MAIN_LOG = "conversion_main.log"
+FAILED_LOG = "conversion_errors.log"
 
-    data_into_list = data.replace('\n', ' ').split(" ")
+# Setup main logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.FileHandler(MAIN_LOG, mode='w', encoding='utf-8'),
+        logging.StreamHandler()
+    ]
+)
 
+# Setup failed conversion logging (Will overwrite each run, could be replaced with RotatingFileHandler but needs script changes)
+failed_logger = logging.getLogger("failed")
+failed_handler = logging.FileHandler(FAILED_LOG, mode='w', encoding='utf-8')
+failed_handler.setLevel(logging.ERROR)
+failed_logger.addHandler(failed_handler)
+failed_logger.propagate = False
 
-    for i in range(len(data_into_list)):
-        if data_into_list[i].startswith('wem'):
+# Counters for summary
+total_wems = 0
+converted_count = 0
+skipped_count = 0
+failed_count = 0
 
-            wavname = "./txtp/" + data_into_list[i].split('.')[0] + '.wav'
+# Step 1: Convert all .wem files into ./out_temp/wem/ (flat), mapping to digit folders
+def wem_to_wav(input_root, temp_root):
+    global total_wems, converted_count, skipped_count, failed_count
+    input_root = Path(input_root)
+    temp_wem_root = Path(temp_root) / "wem"
 
-            if os.path.isfile(wavname):
-                os.rename(wavname, "./out/" + filename.split('.')[0] + '_' + str(i) + '.wav')
+    # CLEAN temp folder
+    if temp_wem_root.exists():
+        shutil.rmtree(temp_wem_root)
+    temp_wem_root.mkdir(parents=True, exist_ok=True)
+
+    mapping = {}  # wav filename -> digit folder
+
+    for folder, _, files in os.walk(input_root):
+        folder_path = Path(folder)
+
+        # If we are in root (txtp/wem) use "root" as folder name
+        digit_folder = "root" if folder_path == input_root else folder_path.name
+
+        for file in files:
+            ext = Path(file).suffix.lower()
+            base_name = Path(file).stem
+            wav_name = base_name + ".wav"
+
+            wem_path = folder_path / file
+            wav_path = temp_wem_root / wav_name
+            mapping[wav_name] = digit_folder
+
+            final_out_path = Path("out") / digit_folder / wav_name
+            if wav_path.exists() or final_out_path.exists():
+                skipped_count += 1
+                logging.info(f"Skipping existing WAV: {wav_path} or {final_out_path}")
+                continue
+
+            if ext == ".wem":
+                # Convert wem → wav
+                logging.info(f"Converting: {wem_path} → {wav_path}")
+                result = subprocess.run(
+                    [str(VGMSTREAM), "-o", str(wav_path), str(wem_path)],
+                    capture_output=True,
+                    text=True
+                )
+                if result.returncode != 0 or not wav_path.exists():
+                    failed_count += 1
+                    logging.error(f"Conversion failed for {wem_path}: {result.stderr}")
+                    failed_logger.error(str(wem_path))
+                else:
+                    converted_count += 1
+                    logging.info(f"Converted {wem_path} successfully")
+
+            elif ext == ".wav":
+                # Copy pre-existing wav into temp for rename step
+                try:
+                    shutil.copy2(wem_path, wav_path)
+                    skipped_count += 1
+                    logging.info(f"Using existing WAV instead of converting: {wem_path} → {wav_path}")
+                except Exception as e:
+                    failed_count += 1
+                    logging.error(f"Failed to copy existing WAV {wem_path}: {e}")
+                    failed_logger.error(str(wem_path))
+    return mapping
+
+# Step 2: Rename .wav files based on .txtp references
+def convert(filename, wav_root, out_root, mapping):
+    wav_root = Path(wav_root)
+    out_root = Path(out_root)
+    txtp_path = Path("txtp") / filename
+
+    try:
+        with open(txtp_path, "r", encoding='utf-8') as my_file:
+            data = my_file.read()
+    except Exception as e:
+        logging.error(f"Failed to read {txtp_path}: {e}")
+        return
+
+    tokens = data.replace('\n', ' ').split(" ")
+
+    for i, token in enumerate(tokens):
+        if token.startswith('wem'):
+            wav_file_only = Path(token).stem + ".wav"
+            wavname = wav_root / wav_file_only
+            digit_folder = mapping.get(wavname.name, "unknown")
+            out_folder = out_root / digit_folder
+            out_folder.mkdir(parents=True, exist_ok=True)
+            new_name = out_folder / f"{filename.split('.')[0]}_{i}.wav"
+
+            if new_name.exists():
+                logging.info(f"Skipping already renamed WAV: {new_name}")
+                continue
+
+            if wavname.exists():
+                try:
+                    shutil.move(str(wavname), str(new_name))
+                    logging.info(f"Renamed {wavname} → {new_name}")
+                except Exception as e:
+                    logging.error(f"Failed to rename {wavname}: {e}")
             else:
-                print(wavname + " not found.")
+                logging.warning(f"{wavname} not found.")
+
+# Step 3: Retry failed conversions
+def retry_failed_conversions(temp_wav_root):
+    global converted_count, failed_count
+    failed_path = Path(FAILED_LOG)
+    if not failed_path.exists():
+        logging.info("No failed conversions to retry.")
+        return
+
+    logging.info("Retrying failed conversions...")
+
+    # Read and truncate the failed log for this retry
+    with open(failed_path, "r+", encoding="utf-8") as f:
+        failed_files = [line.strip() for line in f.readlines() if line.strip()]
+        f.seek(0)
+        f.truncate(0)
+
+    new_failures = 0  # counter for files that fail again
+
+    for wem_path_str in failed_files:
+        wem_path = Path(wem_path_str)
+        wav_name = wem_path.stem + ".wav"
+        wav_path = temp_wav_root / wav_name
+
+        if wav_path.exists():
+            logging.info(f"Skipping existing WAV: {wav_path}")
+            continue
+
+        logging.info(f"Retrying conversion: {wem_path} → {wav_path}")
+        result = subprocess.run(
+            [str(VGMSTREAM), "-o", str(wav_path), str(wem_path)],
+            capture_output=True,
+            text=True
+        )
+        if result.returncode != 0 or not wav_path.exists():
+            new_failures += 1
+            logging.error(
+                f"Conversion failed a 2nd time: {wem_path}. "
+                "Either the .wem file is corrupt, broken, or there is no .txtp path for that file. "
+                "Consider a manual approach or ask for help in the Discord."
+            )
+            failed_logger.error(str(wem_path))
+        else:
+            # Count as converted only if it actually succeeds now
+            converted_count += 1
+            logging.info(f"Successfully converted on retry: {wem_path}")
+
+    # Update failed_count to reflect files that truly failed after retry
+    failed_count = new_failures
+
+# Main driver
+if __name__ == "__main__":
+    wem_root = Path("txtp/wem")
+    wav_temp_root = Path("out_temp") / "wem"
+    out_root = Path("out")
+
+    logging.info("Starting .wem → .wav conversion")
+    mapping = wem_to_wav(wem_root, Path("out_temp"))
 
-    my_file.close()
+    logging.info("Starting .wav renaming based on .txtp files")
+    txtp_files = [f for f in Path("txtp").glob("*.txtp")]
+    for file_path in txtp_files:
+        convert(file_path.name, wav_temp_root, out_root, mapping)
 
-relevant_path = "./txtp/"
-included_extensions = ['txtp']
-file_names = [fn for fn in os.listdir(relevant_path)
-              if any(fn.endswith(ext) for ext in included_extensions)]
+    # Retry any failed conversions
+    retry_failed_conversions(wav_temp_root)
 
+    # Clean up temp folder
+    if wav_temp_root.parent.exists():
+        shutil.rmtree(wav_temp_root.parent)
+        logging.info(f"Temporary folder {wav_temp_root.parent} deleted")
 
-for file_name in file_names:
-    convert(file_name)
+    # Final summary
+    logging.info("===================================")
+    logging.info(f"Total .wem files found: {total_wems}")
+    logging.info(f"Successfully converted: {converted_count}")
+    logging.info(f"Skipped (already exists): {skipped_count}")
+    logging.info(f"Failed conversions: {failed_count}")
+    logging.info("Conversion and renaming complete")
+    logging.info("===================================")

modules/ROOT/pages/Development/ExtractGameFiles.adoc

@@ -104,7 +104,7 @@ Press `OK` to save your changes.
 
 image:ExtractingGameFiles/FModelModelSettings.png[FModel Model Export Settings]
 
-[WARNING] 
+[WARNING]
 ====
 Any other changes made are at user discretion, and dependent on level of knowledge.
 ====
@@ -245,7 +245,7 @@ go back to grab the `wwnames.db3` from the GitHub release and put it in the same
 // cspell:ignore txtp
 Next, select `Generate TXTP` which will create a folder in the same directory as the bnk file
 containing a txtp file for the event.
-// Need the + symbols to make sure Asciidoc doesn't see them as attributes 
+// Need the + symbols to make sure Asciidoc doesn't see them as attributes
 (ex. `+Play_EQ_JetPack_Activate {s} {m}.txtp+`)
 
 Open the txtp file in a text editor of your choice.
@@ -272,6 +272,14 @@ After optionally previewing the sound file in the player,
 right click on it in the player's playlist and select Save,
 prompting a system dialog to select a save location.
 
+[TIP]
+====
+You can also select the entire `Media` folder to extract all sound files at once. It will export the Audio Files as `.wem` files.
+This will be useful if you need to extract a large number of sounds quickly and works great with the Script mentioned further down.
+
+Be aware though that this method will extract all sounds, including those you may not need, and can result in file sizes exceeding 5 GB. If you only need specific sounds, it's recommended to extract them individually.
+====
+
 [WARNING]
 ====
 Some users have reported issues with FModel's audio player,
@@ -290,17 +298,36 @@ if that didn't work (mod developer discord role required to view).
 === Bulk Audio Renamer
 
 Community member MrCheese has created a python script that enables mass renaming of exported wem files to their associated named bnk files.
-If you decide to extract a large number of sounds, this script can save you a lot of time.
-
-To use it:
+This has since been enhanced by community member Rovetown to fully automate the process. If you decide to extract a large number of sounds, this script can save you a lot of time.
 
 1. Create a folder somewhere named `SatisfactoryAudioRenamer`.
 2. Create a subfolder named `out`
 3. Create a subfolder named `txtp`
-4. link:{attachmentsdir}/Development/SatisfactoryAudioRenamer/convert.py[Download this python file (convert.py)]
-   and place it in the SatisfactoryAudioRenamer folder
-5. Move all the txtp files that wwiser generated earlier to the txtp subfolder
+4. Create a subfolder inside `txtp` named `wem`
+5. Place the extracted `.wem` files (including their parent folder if you decided to extract the full folder structure) into the `wem` subfolder.
+6. link:{attachmentsdir}/Development/SatisfactoryAudioRenamer/convert.py[Download this python file (convert.py)]
+   and place it in the SatisfactoryAudioRenamer folder.
+7. Be sure to change the `VGMSTREAM` Path inside the downloaded `converter.py` to the location of your `vgmstream-cli.exe`
+8. Move all the txtp files that wwiser generated earlier to the txtp subfolder
    and run `python .\convert.py` from a terminal in that SatisfactoryAudioRenamer folder.
+9. You now have the renamed `.wav` files in the `out` subfolder (and in their original folder structure).
+
+[NOTE]
+====
+The Script will try and rerun if it encounters any errors.
+However, it may not always succeed in properly renaming all files.
+This can be due to several reasons, so if you encounter this, feel free to ask for help in the discord.
+
+If you get any Errors regarding Python Libraries, make sure you have all the required dependencies installed.
+These are:
+
+* os
+* subprocess
+* pathlib
+* shutil
+* logging
+
+====
 
 == Generating a Complete Starter Project