diff --git a/vast_start.py b/vast_start.py index a41a9af..8991bb0 100644 --- a/vast_start.py +++ b/vast_start.py @@ -126,6 +126,25 @@ def get_instance_details(instance_id: int) -> Dict[str, Any]: raise Exception(f"Instance {instance_id} not found.") return instances +def get_instance_demon_logs(instance_id: int) -> str: + """Fetches the latest logs for an instance.""" + js_data={'daemon_logs': 'true'} + log_meta = vast_request("PUT", f"instances/request_logs/{instance_id}/", json_data=js_data) + temp_download_url = log_meta.get("temp_download_url") + if not temp_download_url: + print(json.dumps(log_meta, indent=2)) + raise Exception(f"Failed to get logs for instance {instance_id}. API Response: {log_meta}") + log_response=None + for _ in range(15): + log_response = requests.get( temp_download_url,timeout=30, headers={"cache-control": "no-cache"}) + if log_response.ok: + break + time.sleep(1) + + if not log_response.ok: + raise Exception(f"Failed to download logs from {temp_download_url}. Status: {log_response.status_code}") + + return log_response.text def get_instances() -> List[Dict[str, Any]]: """Fetches a list of all instances for the user.""" response = vast_request("GET", "instances/") @@ -159,7 +178,13 @@ def wait_until_ready(instance_id: int, poll_interval: int = 15) -> Dict[str, Any return data else: print("Instance running, but waiting for SSH details to populate...") - + print("Instance not ready yet. Fetching logs for insights...") + try: + logs = get_instance_demon_logs(instance_id) + print("Retrieved logs:") + print('\n'.join(logs.splitlines()[-30:])) # Print last 10 lines for brevity + except Exception as e: + print(f"Error fetching logs: {e}") time.sleep(poll_interval) def ssh_and_setup(ip: str, port: int, user: str = "root") -> None: @@ -176,7 +201,7 @@ def ssh_and_setup(ip: str, port: int, user: str = "root") -> None: # Disable auto-tmux subprocess.run(ssh_base_cmd + ["touch", "~/.no_auto_tmux"], check=True) try: - subprocess.run(ssh_base_cmd + ["tmux","new","-d","-s","ollama", "ollama","serve"], check=True) + subprocess.run(ssh_base_cmd + ["tmux","new","-d","-s","ollama", "ollama serve"], check=True) except subprocess.CalledProcessError as e: print(f"Error occurred while setting up tmux session: {e}") # Pull the required Ollama model