Upload folder using huggingface_hub

Bros!_Lora.safetensors → 4mand4_v2-000025.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a24bee799fe9e7536ee913a5da8b8892cd97bc2ca02a2aa3a902a92a2a1a2208
-size 2535602544
+oid sha256:0454a3bce57112fdb9fbad17a364337a3e33ed33976defcf6616ce973682168e
+size 2535602400

4mand4_v2-000050.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b007664bc9ac5bf8665f5e4a0142753345d10f749669ac6e482d03e2a1e0f3b2
+size 2535602400

4mand4_v2-000075.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41caac318cd8fbea59a6877c55a05b6945ee90a48efa88f80a9731c9648881e4
+size 2535602400

4mand4_v2-000100.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e081ba922079b286293f61598d93a69b718d8041b324c9403206d30f57f2632f
+size 2535602400

4mand4_v2-000125.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6ad703e452e15606e1a0c544f6bb8d14da5ee566036f9dd85600bce731ac156
+size 2535602400

Hugging_Face_Upload_Download_V9.ipynb

@@ -0,0 +1,359 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0abc7dd7",
+   "metadata": {},
+   "source": [
+    "### Master Notebook for Hugging Face repos Upload and Download V9\n",
+    "\n",
+    "### Always get latest version before starting to use : https://www.patreon.com/posts/104672510\n",
+    "\n",
+    "### Execute below cell once to install libraries\n",
+    "\n",
+    "### Don't forget to set accurate folder and file paths and repo names befor executing cells"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "caa74bef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install huggingface_hub --upgrade\n",
+    "\n",
+    "!pip install ipywidgets --upgrade\n",
+    "\n",
+    "!pip install git+https://github.com/huggingface/huggingface_hub --upgrade"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "502249bf",
+   "metadata": {},
+   "source": [
+    "### Use below cell to paste your Hugging Face token key. \n",
+    "\n",
+    "### Access Tokens are here : https://huggingface.co/settings/tokens"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0d8027c8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import subprocess\n",
+    "import platform\n",
+    "\n",
+    "hugging_face_token = 'replace_with_your_HF_token'\n",
+    "\n",
+    "# Set the environment variable\n",
+    "os.environ['HUGGING_FACE_HUB_TOKEN'] = hugging_face_token\n",
+    "\n",
+    "# Determine the operating system\n",
+    "system = platform.system()\n",
+    "\n",
+    "if system == \"Linux\":\n",
+    "    export_command = f'export HUGGING_FACE_HUB_TOKEN={hugging_face_token}'\n",
+    "    subprocess.run(export_command, shell=True, check=True)\n",
+    "elif system == \"Windows\":\n",
+    "    set_command = f'set HUGGING_FACE_HUB_TOKEN={hugging_face_token}'\n",
+    "    subprocess.run(set_command, shell=True, check=True)\n",
+    "\n",
+    "# Command to log in using the token\n",
+    "login_command = ['huggingface-cli', 'login', '--token', hugging_face_token]\n",
+    "\n",
+    "# Execute the login command and capture output\n",
+    "try:\n",
+    "    result = subprocess.run(login_command, check=True, capture_output=True, text=True)\n",
+    "    print(\"Output:\", result.stdout)\n",
+    "    print(\"Error:\", result.stderr)\n",
+    "except subprocess.CalledProcessError as e:\n",
+    "    print(\"Command failed with exit code:\", e.returncode)\n",
+    "    print(\"Output:\", e.output)\n",
+    "    print(\"Error:\", e.stderr)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "afb149da",
+   "metadata": {},
+   "source": [
+    "### Very fast new upload - Wait till UPLOAD COMPLETED printed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30b96a4f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!huggingface-cli upload-large-folder \"YourUserName/reponame\" r\"/home/Ubuntu/apps/StableSwarmUI/Models/Lora\" --repo-type=model --no-bars\n",
+    "\n",
+    "print(\".\\n.\\nUPLOAD COMPLETED\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4e7a6316",
+   "metadata": {},
+   "source": [
+    "### Very fast new upload V2 - Wait till UPLOAD COMPLETED printed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75a18901",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from huggingface_hub import HfApi\n",
+    "\n",
+    "api = HfApi()\n",
+    "\n",
+    "api.upload_large_folder(repo_id=\"YourUserName/reponame\", \n",
+    "                        repo_type=\"model\", \n",
+    "                        folder_path=r\"/home/Ubuntu/apps/StableSwarmUI/Models/Lora\")\n",
+    "\n",
+    "print(\".\\n.\\nUPLOAD COMPLETED\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "58da07da-2e0b-4a32-a36d-ddf511800c58",
+   "metadata": {},
+   "source": [
+    "### Upload a single file with specific name to remote repo - Wait till UPLOAD COMPLETED printed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0502703b-8b44-4e58-af2c-82f67d713548",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell is used to upload single file into a repo with certain name\n",
+    "\n",
+    "from huggingface_hub import HfApi\n",
+    "api = HfApi()\n",
+    "api.upload_file(\n",
+    "    path_or_fileobj=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion/model_name.safetensors\",\n",
+    "    path_in_repo=\"model_name.safetensors\",\n",
+    "    repo_id=\"YourUserName/reponame\",\n",
+    "    repo_type=\"model\",\n",
+    ")\n",
+    "\n",
+    "print(\".\\n.\\nUPLOAD COMPLETED\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2dd50994-5595-46a6-b315-595ede0cf922",
+   "metadata": {},
+   "source": [
+    "### Upload a folder fast but not suggested for too big and many files - Wait till UPLOAD COMPLETED printed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a89ece44-8f27-43bb-a7fc-e966409d6f5a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell is used to upload a folder into a repo with single commit\n",
+    "\n",
+    "from huggingface_hub import HfApi\n",
+    "api = HfApi()\n",
+    "api.upload_folder(\n",
+    "    folder_path=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\",\n",
+    "    repo_id=\"YourUserName/reponame\",\n",
+    "    repo_type=\"model\",\n",
+    ")\n",
+    "\n",
+    "print(\".\\n.\\nUPLOAD COMPLETED\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4d26f8d6-31cd-42a5-9477-ba1e77cd271c",
+   "metadata": {},
+   "source": [
+    "### To upload all files in given folder to the target Hugging Face repository use below - Wait till UPLOAD COMPLETED printed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6fc96104",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell uploads a folder into remote repo with multi commit\n",
+    "# Supports continue feature so if gets interrupted you can run again to continue / resume\n",
+    "\n",
+    "from huggingface_hub import HfApi\n",
+    "from huggingface_hub import get_collection, delete_collection_item\n",
+    "from huggingface_hub import upload_file\n",
+    "from huggingface_hub import (\n",
+    "    HfFolder,\n",
+    "    ModelCard,\n",
+    "    ModelCardData,\n",
+    "    create_repo,\n",
+    "    hf_hub_download,\n",
+    "    upload_folder,\n",
+    "    whoami,\n",
+    ")\n",
+    "api = HfApi()\n",
+    "upload_folder(\n",
+    "    folder_path=r\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\",\n",
+    "    repo_id=\"YourUserName/reponame\",\n",
+    "    repo_type=\"model\",\n",
+    "    multi_commits=True,\n",
+    "    multi_commits_verbose=True,\n",
+    ")\n",
+    "\n",
+    "print(\".\\n.\\nUPLOAD COMPLETED\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7381726f",
+   "metadata": {},
+   "source": [
+    "### To download all files in given Hugging Face repository use below - Wait till DOWNLOAD COMPLETED printed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "90f6c9b1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# You can run this cell multiple times if any error occurs it will resume\n",
+    "\n",
+    "from huggingface_hub import snapshot_download\n",
+    "import os\n",
+    "\n",
+    "# Ensure the local directory exists\n",
+    "\n",
+    "\n",
+    "repo_id=\"YourUserName/reponame\"\n",
+    "local_dir=\"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
+    "\n",
+    "os.makedirs(local_dir, exist_ok=True)\n",
+    "\n",
+    "snapshot_download(repo_id=repo_id,local_dir=local_dir)\n",
+    "\n",
+    "print(\".\\n.\\nDOWNLOAD COMPLETED\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "09f0f192-4d6b-4511-8366-6c676cd2404c",
+   "metadata": {},
+   "source": [
+    "### Download a specific sub directory / folder - Wait till DOWNLOAD COMPLETED printed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "download_subdirectory",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell downloads a specific subdirectory from a given repo\n",
+    "\n",
+    "from huggingface_hub import snapshot_download\n",
+    "import os\n",
+    "\n",
+    "# Define the repository and subdirectory\n",
+    "repo_id = \"YourUserName/reponame\"\n",
+    "subdirectory = \"folder_path_in_remote_repo/*\"  # don't delete /* part just change folder path - if you get click to show javascript error ignore it and follow folder it will work\n",
+    "local_dir = \"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
+    "\n",
+    "# Ensure the local directory exists\n",
+    "os.makedirs(local_dir, exist_ok=True)\n",
+    "\n",
+    "# Download only the specific subdirectory\n",
+    "snapshot_download(\n",
+    "    repo_id=repo_id,\n",
+    "    local_dir=local_dir,\n",
+    "    allow_patterns=subdirectory\n",
+    ")\n",
+    "\n",
+    "print(f\"Downloaded {subdirectory} from {repo_id} to {local_dir}\")\n",
+    "print(\".\\n.\\nDOWNLOAD COMPLETED\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "47b04ce9-9653-4e4f-9bd9-1b6ed383f502",
+   "metadata": {},
+   "source": [
+    "### Download a specific file - Wait till DOWNLOAD COMPLETED printed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "download_specific_file",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# This cell downloads a specific file from a given repo\n",
+    "\n",
+    "from huggingface_hub import hf_hub_download\n",
+    "import os\n",
+    "\n",
+    "# Define the repository, file path, and local directory\n",
+    "repo_id = \"YourUserName/reponame\"\n",
+    "file_path = \"folder/file_name.extension\"\n",
+    "local_dir = \"/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion\"\n",
+    "\n",
+    "# Ensure the local directory exists\n",
+    "os.makedirs(local_dir, exist_ok=True)\n",
+    "\n",
+    "# Download the specific file\n",
+    "file_path_local = hf_hub_download(\n",
+    "    repo_id=repo_id,\n",
+    "    filename=file_path,\n",
+    "    local_dir=local_dir\n",
+    ")\n",
+    "\n",
+    "print(f\"Downloaded {file_path} from {repo_id} to {file_path_local}\")\n",
+    "print(\".\\n.\\nDOWNLOAD COMPLETED\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}