xcc.es
[READ-ONLY] Mirror of https://github.com/mrgnw/scripts.
9.2 kB
261 lines
1#!/usr/bin/env -S uv run --script
2# /// script
3# requires-python = ">=3.12"
4# dependencies = [
5# "requests",
6# "aiohttp",
7# "rich"
8# ]
9# ///
10import os
11import requests
12import asyncio
13import aiohttp
14from pathlib import Path
15import sys
16from fnmatch import fnmatch
17from rich.console import Console
18from rich.panel import Panel
19from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn
20
21# Configuration
22DEFAULT_BRANCH = 'main'
23GITHUB_TOKEN = os.getenv('GITHUB_TOKEN')
24OUTPUT_FILE = 'repo_files.md'
25HEADERS = (
26 {'Authorization': f'bearer {GITHUB_TOKEN}', 'Content-Type': 'application/json'} if GITHUB_TOKEN else {}
27)
28
29# Initialize rich console
30console = Console()
31
32# Svelte project file patterns
33SVELTE_PATTERNS = [
34 'package.json',
35 'svelte.config.js',
36 'vite.config.js',
37 'vite.config.ts',
38 'src/routes/**/*.svelte',
39 'src/routes/**/*.js',
40 'src/routes/**/*.ts',
41 'src/lib/**/*.svelte',
42 'src/lib/**/*.js',
43 'src/lib/**/*.ts',
44 'src/app.html',
45 'src/app.d.ts',
46]
47
48
49async def fetch_repo_files(repo, branch='main'):
50 """Fetch all files matching patterns from a repository."""
51 owner, name = repo.split('/')
52
53 # Check repository and get default branch if needed
54 with console.status("[bold green]Checking repository...", spinner="dots"):
55 url = f"https://api.github.com/repos/{owner}/{name}"
56 response = requests.get(url, headers=HEADERS)
57 response.raise_for_status()
58 repo_info = response.json()
59
60 if branch == 'main' and repo_info['default_branch'] != 'main':
61 branch = repo_info['default_branch']
62 console.print(f"[yellow]Using default branch:[/] [bold cyan]{branch}[/]")
63
64 # Get repository contents using Git Trees API
65 with console.status("[bold green]Fetching repository structure...", spinner="dots"):
66 tree_url = f"https://api.github.com/repos/{owner}/{name}/git/trees/{branch}?recursive=1"
67 response = requests.get(tree_url, headers=HEADERS)
68 response.raise_for_status()
69 tree_data = response.json()
70
71 if tree_data.get('truncated', False):
72 console.print("[yellow]Warning: Repository tree is truncated. Some files may be missing.[/]")
73
74 # Filter files based on patterns
75 matching_files = []
76 for item in tree_data.get('tree', []):
77 if item['type'] == 'blob' and any(fnmatch(item['path'], pattern) for pattern in SVELTE_PATTERNS):
78 matching_files.append({'path': item['path']})
79
80 console.print(f"Found [bold green]{len(matching_files)}[/] matching files.")
81 return matching_files
82
83
84async def fetch_file_contents(repo, files, branch):
85 """Fetch file contents asynchronously with progress tracking."""
86 owner, name = repo.split('/')
87 results = []
88
89 with Progress(
90 TextColumn("[bold blue]{task.description}"),
91 BarColumn(),
92 TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
93 TimeRemainingColumn(),
94 ) as progress:
95 overall_task = progress.add_task(f"[cyan]Downloading files...", total=len(files))
96
97 async with aiohttp.ClientSession() as session:
98 # Process in batches to avoid rate limits
99 batch_size = 5
100 for i in range(0, len(files), batch_size):
101 batch = files[i:i+batch_size]
102 batch_tasks = []
103
104 for file in batch:
105 url = f"https://raw.githubusercontent.com/{owner}/{name}/{branch}/{file['path']}"
106 task = asyncio.create_task(fetch_single_file(session, url, file['path']))
107 batch_tasks.append(task)
108
109 # Wait for batch to complete
110 batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True)
111
112 for j, result in enumerate(batch_results):
113 file_path = batch[j]['path']
114 if isinstance(result, Exception):
115 console.print(f"[red]Error fetching {file_path}: {result}[/]")
116 else:
117 results.append({'path': file_path, 'content': result})
118
119 progress.update(overall_task, advance=len(batch))
120
121 return results
122
123
124async def fetch_single_file(session, url, path):
125 """Fetch a single file's content."""
126 async with session.get(url) as response:
127 if response.status != 200:
128 raise ValueError(f"Failed to fetch file: {response.status}")
129 return await response.text()
130
131
132def generate_tree_structure(files):
133 """Generate a tree structure for the output markdown."""
134 # Build directory tree dictionary
135 path_dict = {}
136 for file in files:
137 parts = file['path'].split('/')
138 current = path_dict
139 for i, part in enumerate(parts):
140 if i == len(parts) - 1: # File
141 if 'files' not in current:
142 current['files'] = []
143 current['files'].append(part)
144 else: # Directory
145 if part not in current:
146 current[part] = {}
147 current = current[part]
148
149 # Build markdown tree representation
150 lines = []
151
152 def build_tree(node, prefix=""):
153 items = sorted([(k, v) for k, v in node.items() if k != "files"])
154 files = node.get("files", [])
155
156 # Process directories
157 for i, (name, contents) in enumerate(items):
158 is_last = (i == len(items) - 1 and not files)
159 if is_last:
160 lines.append(f"{prefix}└── 📁 {name}/")
161 build_tree(contents, prefix + " ")
162 else:
163 lines.append(f"{prefix}├── 📁 {name}/")
164 build_tree(contents, prefix + "│ ")
165
166 # Process files
167 for i, file in enumerate(sorted(files)):
168 is_last = (i == len(files) - 1)
169 ext = file.split('.')[-1] if '.' in file else ''
170
171 icon = "📄"
172 if ext in ['js', 'ts']:
173 icon = "🟨"
174 elif ext == 'json':
175 icon = "🔧"
176 elif ext == 'svelte':
177 icon = "🔥"
178
179 if is_last:
180 lines.append(f"{prefix}└── {icon} {file}")
181 else:
182 lines.append(f"{prefix}├── {icon} {file}")
183
184 build_tree(path_dict)
185 return "\n".join(lines)
186
187
188async def main():
189 # Parse command line arguments
190 if len(sys.argv) < 2:
191 console.print("[bold red]Error:[/] Please provide a repository name (owner/repo) or URL")
192 sys.exit(1)
193
194 # Parse repo from argument (handle URLs too)
195 repo_arg = sys.argv[1]
196 if repo_arg.startswith("https://"):
197 parts = repo_arg.strip('/').split('/')
198 if len(parts) >= 4 and parts[2] in ['github.com', 'www.github.com']:
199 repo = f"{parts[-2]}/{parts[-1]}"
200 else:
201 console.print("[bold red]Error:[/] Invalid GitHub URL format")
202 sys.exit(1)
203 else:
204 repo = repo_arg
205
206 # Get output file path (optional second argument)
207 output_file = sys.argv[2] if len(sys.argv) > 2 else OUTPUT_FILE
208
209 # Get branch (optional --branch argument)
210 branch = DEFAULT_BRANCH
211 if "--branch" in sys.argv and sys.argv.index("--branch") + 1 < len(sys.argv):
212 branch_idx = sys.argv.index("--branch") + 1
213 branch = sys.argv[branch_idx]
214
215 console.print(Panel.fit(
216 f"[bold cyan]Svelte Repository Explorer[/]\n[green]{repo}[/] (branch: [yellow]{branch}[/])",
217 border_style="cyan"
218 ))
219
220 try:
221 # Fetch matching files
222 files = await fetch_repo_files(repo, branch)
223 if not files:
224 console.print("[yellow]No Svelte project files found in repository.[/]")
225 return
226
227 # Sort files for consistent output
228 files.sort(key=lambda f: f['path'])
229
230 # Fetch file contents
231 file_contents = await fetch_file_contents(repo, files, branch)
232 console.print(f"Successfully fetched [bold green]{len(file_contents)}/{len(files)}[/] files")
233
234 # Generate tree structure
235 tree_md = generate_tree_structure(files)
236
237 # Generate markdown content
238 repo_name = repo.split('/')[1]
239 md_content = f'# {repo_name} Project Structure\n\n'
240 md_content += "## File Tree\n\n"
241 md_content += tree_md
242 md_content += "\n\n## File Contents\n\n"
243
244 # Add file contents
245 for file in file_contents:
246 file_path = file['path']
247 content = file['content']
248 ext = file_path.split('.')[-1] if '.' in file_path else ''
249 md_content += f"### {file_path}\n\n```{ext}\n{content}\n```\n\n"
250
251 # Write to file
252 Path(output_file).write_text(md_content, encoding='utf-8')
253 console.print(f"[bold green]Generated:[/] {output_file}")
254
255 except Exception as e:
256 console.print(f"[bold red]Error:[/] {e}")
257 sys.exit(1)
258
259
260if __name__ == '__main__':
261 asyncio.run(main())