[READ-ONLY] Mirror of https://github.com/mrgnw/scripts.
0

Configure Feed

Select the types of activity you want to include in your feed.

cleanup

+154 -340
+154 -340
gitllm
··· 3 3 # requires-python = ">=3.12" 4 4 # dependencies = [ 5 5 # "requests", 6 - # "tqdm", 7 - # "argparse", 8 6 # "aiohttp", 9 - # "asyncio", 10 7 # "rich" 11 8 # ] 12 9 # /// 13 10 import os 14 11 import requests 15 - import argparse 16 12 import asyncio 17 13 import aiohttp 18 14 from pathlib import Path ··· 20 16 from fnmatch import fnmatch 21 17 from rich.console import Console 22 18 from rich.panel import Panel 23 - from rich.syntax import Syntax 24 - from rich.markdown import Markdown 25 - from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn, TimeRemainingColumn 26 - from rich.tree import Tree 27 - from rich import print as rprint 28 - from tqdm.asyncio import tqdm as async_tqdm 19 + from rich.progress import Progress, TextColumn, BarColumn, TimeRemainingColumn 29 20 30 21 # Configuration 31 22 DEFAULT_BRANCH = 'main' 32 - GITHUB_TOKEN = os.getenv('GITHUB_TOKEN') # Set your GitHub token in env variables 33 - OUTPUT_FILE = 'repo_files.md' # Updated default file name 23 + GITHUB_TOKEN = os.getenv('GITHUB_TOKEN') 24 + OUTPUT_FILE = 'repo_files.md' 34 25 HEADERS = ( 35 26 {'Authorization': f'bearer {GITHUB_TOKEN}', 'Content-Type': 'application/json'} if GITHUB_TOKEN else {} 36 27 ) ··· 38 29 # Initialize rich console 39 30 console = Console() 40 31 41 - # File patterns to fetch 42 - INCLUDE_PATTERNS = [ 32 + # Svelte project file patterns 33 + SVELTE_PATTERNS = [ 43 34 'package.json', 44 - '**/*.config.js', 45 - '**/*.config.ts', 35 + 'svelte.config.js', 36 + 'vite.config.js', 37 + 'vite.config.ts', 46 38 'src/routes/**/*.svelte', 47 39 'src/routes/**/*.js', 48 40 'src/routes/**/*.ts', 49 - # Adding more general patterns for all JavaScript, TypeScript, and Svelte files 50 - '**/*.js', 51 - '**/*.ts', 52 - '**/*.svelte', 53 - '**/*.json' 41 + 'src/lib/**/*.svelte', 42 + 'src/lib/**/*.js', 43 + 'src/lib/**/*.ts', 44 + 'src/app.html', 45 + 'src/app.d.ts', 54 46 ] 55 47 56 48 57 - def graphql_query(query, variables=None): 58 - url = 'https://api.github.com/graphql' 59 - payload = {'query': query, 'variables': variables or {}} 60 - response = requests.post(url, headers=HEADERS, json=payload) 61 - response.raise_for_status() 62 - return response.json() 63 - 64 - 65 - async def async_graphql_query(session, query, variables=None): 66 - url = 'https://api.github.com/graphql' 67 - payload = {'query': query, 'variables': variables or {}} 68 - async with session.post(url, json=payload, headers=HEADERS) as response: 69 - response.raise_for_status() 70 - return await response.json() 71 - 72 - 73 - def get_all_files(repo, branch='main'): 74 - """Get all files in the repository using a single GraphQL query with Git tree.""" 49 + async def fetch_repo_files(repo, branch='main'): 50 + """Fetch all files matching patterns from a repository.""" 75 51 owner, name = repo.split('/') 76 52 77 - # First, try to get the default branch 78 - with console.status(f"[bold green]Checking repository {repo}...", spinner="dots"): 79 - query = """ 80 - query($owner: String!, $name: String!) { 81 - repository(owner: $owner, name: $name) { 82 - defaultBranchRef { 83 - name 84 - } 85 - } 86 - } 87 - """ 88 - 89 - data = graphql_query(query, {'owner': owner, 'name': name}) 53 + # Check repository and get default branch if needed 54 + with console.status("[bold green]Checking repository...", spinner="dots"): 55 + url = f"https://api.github.com/repos/{owner}/{name}" 56 + response = requests.get(url, headers=HEADERS) 57 + response.raise_for_status() 58 + repo_info = response.json() 90 59 91 - if 'data' not in data or data['data'] is None or not data['data'].get('repository'): 92 - raise ValueError(f"Error fetching repository data. API response: {data}") 93 - 94 - repository = data['data'].get('repository') 95 - 96 - # Use default branch if main is not available 97 - if branch == 'main' and repository.get('defaultBranchRef'): 98 - default_branch = repository['defaultBranchRef']['name'] 99 - if default_branch != branch: 100 - console.print(f"[yellow]Using default branch:[/] [bold cyan]{default_branch}[/]") 101 - branch = default_branch 60 + if branch == 'main' and repo_info['default_branch'] != 'main': 61 + branch = repo_info['default_branch'] 62 + console.print(f"[yellow]Using default branch:[/] [bold cyan]{branch}[/]") 102 63 103 - # Use Git Trees API directly - more reliable for full repository listing 104 - rest_api_url = f"https://api.github.com/repos/{owner}/{name}/git/trees/{branch}?recursive=1" 105 - console.print(f"[bold]Fetching file list from:[/] [blue]{rest_api_url}[/]") 106 - 107 - with console.status("[bold green]Downloading repository structure...", spinner="dots"): 108 - response = requests.get(rest_api_url, headers=HEADERS) 64 + # Get repository contents using Git Trees API 65 + with console.status("[bold green]Fetching repository structure...", spinner="dots"): 66 + tree_url = f"https://api.github.com/repos/{owner}/{name}/git/trees/{branch}?recursive=1" 67 + response = requests.get(tree_url, headers=HEADERS) 109 68 response.raise_for_status() 110 - 111 - all_files = [] 112 69 tree_data = response.json() 113 70 114 71 if tree_data.get('truncated', False): 115 - console.print("[bold yellow]Warning:[/] Repository tree is too large and was truncated. Some files may be missing.") 116 - 117 - # Print total number of files found 118 - total_files = len(tree_data.get('tree', [])) 119 - console.print(f"Total files in repository: [bold cyan]{total_files}[/]") 72 + console.print("[yellow]Warning: Repository tree is truncated. Some files may be missing.[/]") 120 73 121 - # Process all blob entries (files) 74 + # Filter files based on patterns 75 + matching_files = [] 122 76 for item in tree_data.get('tree', []): 123 - if item['type'] == 'blob': # Only include files, not directories 124 - all_files.append({ 125 - 'path': item['path'], 126 - 'type': item['type'] 127 - }) 128 - 129 - console.print(f"Total blob entries: [bold cyan]{len(all_files)}[/]") 77 + if item['type'] == 'blob' and any(fnmatch(item['path'], pattern) for pattern in SVELTE_PATTERNS): 78 + matching_files.append({'path': item['path']}) 130 79 131 - # Filter files based on patterns 132 - matching_files = [] 133 - for file in all_files: 134 - for pattern in INCLUDE_PATTERNS: 135 - if fnmatch(file['path'], pattern): 136 - matching_files.append(file) 137 - break # No need to check other patterns 138 - 139 - # Display matching files as a tree 140 - file_tree = Tree("[bold]Matching files found:[/]") 141 - 142 - # Group files by directory for better visualization 143 - file_paths = [file['path'] for file in matching_files] 144 - file_paths.sort() 145 - 146 - # Build directory tree 147 - path_dict = {} 148 - for path in file_paths: 149 - parts = path.split('/') 150 - current_dict = path_dict 151 - for i, part in enumerate(parts): 152 - if i == len(parts) - 1: # If it's the last part (file) 153 - if 'files' not in current_dict: 154 - current_dict['files'] = [] 155 - current_dict['files'].append(part) 156 - else: # If it's a directory 157 - if part not in current_dict: 158 - current_dict[part] = {} 159 - current_dict = current_dict[part] 160 - 161 - # Function to build tree display 162 - def build_tree(tree_node, path_dict, is_root=False): 163 - # Add directories first 164 - for key, value in sorted(path_dict.items()): 165 - if key != 'files': 166 - dir_node = tree_node.add(f"[bold blue]{key}/[/]") 167 - build_tree(dir_node, value) 168 - 169 - # Then add files 170 - if 'files' in path_dict: 171 - for file in sorted(path_dict['files']): 172 - ext = file.split('.')[-1] if '.' in file else '' 173 - if ext in ['js', 'ts']: 174 - tree_node.add(f"[yellow]{file}[/]") 175 - elif ext == 'json': 176 - tree_node.add(f"[green]{file}[/]") 177 - elif ext == 'svelte': 178 - tree_node.add(f"[orange1]{file}[/]") 179 - else: 180 - tree_node.add(f"[white]{file}[/]") 181 - 182 - # Build and display the tree 183 - build_tree(file_tree, path_dict, True) 184 - console.print(file_tree) 185 - 186 - console.print(f"\nFound [bold green]{len(matching_files)}[/] matching files. Fetching content...") 187 - 188 - # Generate a markdown representation of the tree for the output file 189 - # Use GitHub-flavored markdown tree format 190 - md_tree_content = [] 191 - 192 - def build_md_tree(path_dict, prefix=""): 193 - items = sorted([(k, v) for k, v in path_dict.items() if k != "files"]) 194 - file_items = path_dict.get("files", []) 195 - 196 - # Process all items except the last one with ├── prefix 197 - for i, (key, value) in enumerate(items[:-1] if items else []): 198 - md_tree_content.append(f"{prefix}├── 📁 {key}/") 199 - new_prefix = f"{prefix}│ " 200 - build_md_tree(value, new_prefix) 201 - 202 - # Process the last directory item with └── prefix 203 - if items: 204 - key, value = items[-1] 205 - md_tree_content.append(f"{prefix}└── 📁 {key}/") 206 - new_prefix = f"{prefix} " 207 - build_md_tree(value, new_prefix) 208 - 209 - # Process files with appropriate prefixes 210 - sorted_files = sorted(file_items) 211 - for i, file in enumerate(sorted_files): 212 - ext = file.split('.')[-1] if '.' in file else '' 213 - icon = "📄" 214 - if ext in ['js', 'ts']: 215 - icon = "🟨" 216 - elif ext == 'json': 217 - icon = "🔧" 218 - elif ext == 'svelte': 219 - icon = "🔥" 220 - elif ext in ['md', 'markdown']: 221 - icon = "📝" 222 - 223 - # Use different prefix for last item 224 - if i == len(sorted_files) - 1: 225 - md_tree_content.append(f"{prefix}└── {icon} {file}") 226 - else: 227 - md_tree_content.append(f"{prefix}├── {icon} {file}") 228 - 229 - build_md_tree(path_dict) 230 - md_tree = "\n".join(md_tree_content) 231 - 232 - return matching_files, md_tree 80 + console.print(f"Found [bold green]{len(matching_files)}[/] matching files.") 81 + return matching_files 233 82 234 83 235 - async def fetch_content_async(repo, files, branch): 236 - """Fetch file contents asynchronously with individual progress bars for each file.""" 84 + async def fetch_file_contents(repo, files, branch): 85 + """Fetch file contents asynchronously with progress tracking.""" 237 86 owner, name = repo.split('/') 238 - 239 - # Set up a rich progress display with multiple tasks 240 87 results = [] 241 88 242 - # Create multi-progress display 243 89 with Progress( 244 90 TextColumn("[bold blue]{task.description}"), 245 91 BarColumn(), 246 92 TextColumn("[progress.percentage]{task.percentage:>3.0f}%"), 247 - TextColumn("•"), 248 93 TimeRemainingColumn(), 249 - console=console, 250 94 ) as progress: 251 - # Create a task for each file 252 - tasks = {} 253 - for i, file in enumerate(files): 254 - # Truncate file name if it's too long for display 255 - display_name = file['path'] 256 - if len(display_name) > 40: 257 - display_name = "..." + display_name[-37:] 258 - 259 - tasks[file['path']] = progress.add_task( 260 - f"[cyan]{display_name}[/]", 261 - total=1.0, 262 - completed=0.0 263 - ) 95 + overall_task = progress.add_task(f"[cyan]Downloading files...", total=len(files)) 264 96 265 - # Process files with proper progress tracking 266 97 async with aiohttp.ClientSession() as session: 267 - # Process in batches to avoid hitting rate limits 268 - batch_size = 5 # Adjust based on API rate limits 98 + # Process in batches to avoid rate limits 99 + batch_size = 5 269 100 for i in range(0, len(files), batch_size): 270 101 batch = files[i:i+batch_size] 271 - 272 - # Create tasks for each file in the batch 273 102 batch_tasks = [] 103 + 274 104 for file in batch: 275 - task = asyncio.create_task( 276 - fetch_file_with_progress( 277 - session, repo, file['path'], branch, 278 - progress, tasks[file['path']] 279 - ) 280 - ) 105 + url = f"https://raw.githubusercontent.com/{owner}/{name}/{branch}/{file['path']}" 106 + task = asyncio.create_task(fetch_single_file(session, url, file['path'])) 281 107 batch_tasks.append(task) 282 108 283 - # Wait for this batch to complete 109 + # Wait for batch to complete 284 110 batch_results = await asyncio.gather(*batch_tasks, return_exceptions=True) 285 111 286 - # Process results from this batch 287 112 for j, result in enumerate(batch_results): 288 113 file_path = batch[j]['path'] 289 114 if isinstance(result, Exception): 290 115 console.print(f"[red]Error fetching {file_path}: {result}[/]") 291 - # Ensure progress bar is completed even on error 292 - progress.update(tasks[file_path], completed=1.0) 293 116 else: 294 - results.append({ 295 - 'path': file_path, 296 - 'content': result 297 - }) 117 + results.append({'path': file_path, 'content': result}) 118 + 119 + progress.update(overall_task, advance=len(batch)) 298 120 299 121 return results 300 122 301 123 302 - async def fetch_file_with_progress(session, repo, path, branch, progress, task_id): 303 - """Fetch a single file with progress tracking.""" 304 - # First update to show we're starting 305 - progress.update(task_id, completed=0.1) 306 - 307 - try: 308 - # Fetch the file content 309 - content = await fetch_file_content_async(session, repo, path, branch) 310 - # Mark as complete 311 - progress.update(task_id, completed=1.0) 312 - return content 313 - except Exception as e: 314 - progress.update(task_id, completed=1.0) 315 - raise e 124 + async def fetch_single_file(session, url, path): 125 + """Fetch a single file's content.""" 126 + async with session.get(url) as response: 127 + if response.status != 200: 128 + raise ValueError(f"Failed to fetch file: {response.status}") 129 + return await response.text() 316 130 317 131 318 - async def fetch_file_content_async(session, repo, path, branch='main'): 319 - """Fetch content of a specific file asynchronously.""" 320 - query = """ 321 - query($owner: String!, $name: String!, $expression: String!) { 322 - repository(owner: $owner, name: $name) { 323 - object(expression: $expression) { 324 - ... on Blob { 325 - text 326 - } 327 - } 328 - } 329 - } 330 - """ 331 - owner, name = repo.split('/') 332 - expression = f"{branch}:{path}" 333 - data = await async_graphql_query(session, query, {'owner': owner, 'name': name, 'expression': expression}) 132 + def generate_tree_structure(files): 133 + """Generate a tree structure for the output markdown.""" 134 + # Build directory tree dictionary 135 + path_dict = {} 136 + for file in files: 137 + parts = file['path'].split('/') 138 + current = path_dict 139 + for i, part in enumerate(parts): 140 + if i == len(parts) - 1: # File 141 + if 'files' not in current: 142 + current['files'] = [] 143 + current['files'].append(part) 144 + else: # Directory 145 + if part not in current: 146 + current[part] = {} 147 + current = current[part] 334 148 335 - # Check if file exists 336 - if ('data' not in data or 337 - not data['data'].get('repository') or 338 - not data['data']['repository'].get('object')): 339 - raise ValueError(f"File '{path}' not found") 149 + # Build markdown tree representation 150 + lines = [] 340 151 341 - return data['data']['repository']['object'].get('text', '') 152 + def build_tree(node, prefix=""): 153 + items = sorted([(k, v) for k, v in node.items() if k != "files"]) 154 + files = node.get("files", []) 155 + 156 + # Process directories 157 + for i, (name, contents) in enumerate(items): 158 + is_last = (i == len(items) - 1 and not files) 159 + if is_last: 160 + lines.append(f"{prefix}└── 📁 {name}/") 161 + build_tree(contents, prefix + " ") 162 + else: 163 + lines.append(f"{prefix}├── 📁 {name}/") 164 + build_tree(contents, prefix + "│ ") 165 + 166 + # Process files 167 + for i, file in enumerate(sorted(files)): 168 + is_last = (i == len(files) - 1) 169 + ext = file.split('.')[-1] if '.' in file else '' 170 + 171 + icon = "📄" 172 + if ext in ['js', 'ts']: 173 + icon = "🟨" 174 + elif ext == 'json': 175 + icon = "🔧" 176 + elif ext == 'svelte': 177 + icon = "🔥" 178 + 179 + if is_last: 180 + lines.append(f"{prefix}└── {icon} {file}") 181 + else: 182 + lines.append(f"{prefix}├── {icon} {file}") 183 + 184 + build_tree(path_dict) 185 + return "\n".join(lines) 342 186 343 187 344 - def parse_args(): 345 - parser = argparse.ArgumentParser(description='Generate a structure document from GitHub repository files') 346 - parser.add_argument('repo', nargs='?', help='GitHub repository in format owner/repo') 347 - parser.add_argument('output', nargs='?', default=OUTPUT_FILE, help='Output markdown file path') 348 - parser.add_argument('--repo', dest='repo_option', help='GitHub repository in format owner/repo') 349 - parser.add_argument('--branch', default='main', help='Repository branch (default: main)') 350 - parser.add_argument('--patterns', nargs='+', help='File patterns to include') 351 - parser.add_argument('--all', action='store_true', help='Include all js/ts/svelte/json files') 352 - parser.add_argument('--preview', action='store_true', help='Preview the markdown in terminal') 353 - args = parser.parse_args() 354 - 355 - # Use the positional argument if provided, otherwise use the named argument 356 - if not args.repo and not args.repo_option: 357 - parser.error("Repository name is required. Provide it as a positional argument or with --repo") 358 - 359 - # Positional argument takes precedence 360 - final_args = args 361 - final_args.repo = args.repo or args.repo_option 362 - delattr(final_args, 'repo_option') 188 + async def main(): 189 + # Parse command line arguments 190 + if len(sys.argv) < 2: 191 + console.print("[bold red]Error:[/] Please provide a repository name (owner/repo) or URL") 192 + sys.exit(1) 193 + 194 + # Parse repo from argument (handle URLs too) 195 + repo_arg = sys.argv[1] 196 + if repo_arg.startswith("https://"): 197 + parts = repo_arg.strip('/').split('/') 198 + if len(parts) >= 4 and parts[2] in ['github.com', 'www.github.com']: 199 + repo = f"{parts[-2]}/{parts[-1]}" 200 + else: 201 + console.print("[bold red]Error:[/] Invalid GitHub URL format") 202 + sys.exit(1) 203 + else: 204 + repo = repo_arg 363 205 364 - return final_args 365 - 366 - 367 - async def async_main(): 368 - args = parse_args() 206 + # Get output file path (optional second argument) 207 + output_file = sys.argv[2] if len(sys.argv) > 2 else OUTPUT_FILE 369 208 370 - if not args.repo or '/' not in args.repo: 371 - console.print("[bold red]Error:[/] Please provide a valid repository in the format 'owner/repo'") 372 - sys.exit(1) 209 + # Get branch (optional --branch argument) 210 + branch = DEFAULT_BRANCH 211 + if "--branch" in sys.argv and sys.argv.index("--branch") + 1 < len(sys.argv): 212 + branch_idx = sys.argv.index("--branch") + 1 213 + branch = sys.argv[branch_idx] 373 214 374 - # Show title 375 215 console.print(Panel.fit( 376 - f"[bold cyan]GitHub Repository Explorer[/]\n[green]{args.repo}[/] (branch: [yellow]{args.branch}[/])", 216 + f"[bold cyan]Svelte Repository Explorer[/]\n[green]{repo}[/] (branch: [yellow]{branch}[/])", 377 217 border_style="cyan" 378 218 )) 379 219 380 - global INCLUDE_PATTERNS 381 - if args.patterns: 382 - INCLUDE_PATTERNS = args.patterns 383 - elif args.all: 384 - # Use simplified patterns if --all flag is provided 385 - INCLUDE_PATTERNS = [ 386 - '**/*.js', 387 - '**/*.ts', 388 - '**/*.svelte', 389 - '**/*.json' 390 - ] 391 - 392 - output_file = args.output 393 - 394 220 try: 395 - console.print(f"[bold]Fetching repository structure for [green]{args.repo}[/] (branch: [yellow]{args.branch}[/])...") 396 - matching_files, md_tree = get_all_files(args.repo, args.branch) 397 - 398 - if not matching_files: 399 - console.print("[yellow]No files matched the specified patterns.[/]") 221 + # Fetch matching files 222 + files = await fetch_repo_files(repo, branch) 223 + if not files: 224 + console.print("[yellow]No Svelte project files found in repository.[/]") 400 225 return 401 226 402 - # No need to repeat the file count here - removed redundant message 227 + # Sort files for consistent output 228 + files.sort(key=lambda f: f['path']) 403 229 404 - # Process files in sorted order for consistent output 405 - matching_files.sort(key=lambda f: f['path']) 230 + # Fetch file contents 231 + file_contents = await fetch_file_contents(repo, files, branch) 232 + console.print(f"Successfully fetched [bold green]{len(file_contents)}/{len(files)}[/] files") 406 233 407 - # Fetch file contents asynchronously 408 - file_contents = await fetch_content_async(args.repo, matching_files, args.branch) 234 + # Generate tree structure 235 + tree_md = generate_tree_structure(files) 409 236 410 - # Check if we actually got content for all files - simplified message 411 - console.print(f"Successfully fetched [bold green]{len(file_contents)}/{len(matching_files)}[/] files") 412 - 413 - # Extract repository name for the header 414 - repo_name = args.repo.split('/')[1] 415 - 416 - # Generate markdown with tree structure at the top - more GitHub-friendly format 237 + # Generate markdown content 238 + repo_name = repo.split('/')[1] 417 239 md_content = f'# {repo_name} Project Structure\n\n' 418 240 md_content += "## File Tree\n\n" 419 - md_content += md_tree 241 + md_content += tree_md 420 242 md_content += "\n\n## File Contents\n\n" 421 243 422 - for file_data in file_contents: 423 - file_path = file_data['path'] 424 - content = file_data['content'] 425 - # Add language-specific syntax highlighting 426 - extension = file_path.split('.')[-1] if '.' in file_path else '' 427 - md_content += f"### {file_path}\n\n```{extension}\n{content}\n```\n\n" 244 + # Add file contents 245 + for file in file_contents: 246 + file_path = file['path'] 247 + content = file['content'] 248 + ext = file_path.split('.')[-1] if '.' in file_path else '' 249 + md_content += f"### {file_path}\n\n```{ext}\n{content}\n```\n\n" 428 250 429 - Path(args.output).write_text(md_content, encoding='utf-8') 430 - console.print(f"[bold green]Generated:[/] {args.output}") 251 + # Write to file 252 + Path(output_file).write_text(md_content, encoding='utf-8') 253 + console.print(f"[bold green]Generated:[/] {output_file}") 431 254 432 - if args.preview: 433 - console.print("\n[bold]Preview of the generated document:[/]") 434 - md = Markdown(md_content[:1000] + "...\n\n(Output truncated for preview)") 435 - console.print(md) 436 - 437 255 except Exception as e: 438 256 console.print(f"[bold red]Error:[/] {e}") 439 257 sys.exit(1) 440 258 441 259 442 - def main(): 443 - asyncio.run(async_main()) 444 - 445 - 446 260 if __name__ == '__main__': 447 - main() 261 + asyncio.run(main())