feat: add function to fetch instance daemon logs and integrate log retrieval during instance readiness check
This commit is contained in:
@@ -126,6 +126,25 @@ def get_instance_details(instance_id: int) -> Dict[str, Any]:
|
||||
raise Exception(f"Instance {instance_id} not found.")
|
||||
|
||||
return instances
|
||||
def get_instance_demon_logs(instance_id: int) -> str:
|
||||
"""Fetches the latest logs for an instance."""
|
||||
js_data={'daemon_logs': 'true'}
|
||||
log_meta = vast_request("PUT", f"instances/request_logs/{instance_id}/", json_data=js_data)
|
||||
temp_download_url = log_meta.get("temp_download_url")
|
||||
if not temp_download_url:
|
||||
print(json.dumps(log_meta, indent=2))
|
||||
raise Exception(f"Failed to get logs for instance {instance_id}. API Response: {log_meta}")
|
||||
log_response=None
|
||||
for _ in range(15):
|
||||
log_response = requests.get( temp_download_url,timeout=30, headers={"cache-control": "no-cache"})
|
||||
if log_response.ok:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
if not log_response.ok:
|
||||
raise Exception(f"Failed to download logs from {temp_download_url}. Status: {log_response.status_code}")
|
||||
|
||||
return log_response.text
|
||||
def get_instances() -> List[Dict[str, Any]]:
|
||||
"""Fetches a list of all instances for the user."""
|
||||
response = vast_request("GET", "instances/")
|
||||
@@ -159,7 +178,13 @@ def wait_until_ready(instance_id: int, poll_interval: int = 15) -> Dict[str, Any
|
||||
return data
|
||||
else:
|
||||
print("Instance running, but waiting for SSH details to populate...")
|
||||
|
||||
print("Instance not ready yet. Fetching logs for insights...")
|
||||
try:
|
||||
logs = get_instance_demon_logs(instance_id)
|
||||
print("Retrieved logs:")
|
||||
print('\n'.join(logs.splitlines()[-30:])) # Print last 10 lines for brevity
|
||||
except Exception as e:
|
||||
print(f"Error fetching logs: {e}")
|
||||
time.sleep(poll_interval)
|
||||
|
||||
def ssh_and_setup(ip: str, port: int, user: str = "root") -> None:
|
||||
@@ -176,7 +201,7 @@ def ssh_and_setup(ip: str, port: int, user: str = "root") -> None:
|
||||
# Disable auto-tmux
|
||||
subprocess.run(ssh_base_cmd + ["touch", "~/.no_auto_tmux"], check=True)
|
||||
try:
|
||||
subprocess.run(ssh_base_cmd + ["tmux","new","-d","-s","ollama", "ollama","serve"], check=True)
|
||||
subprocess.run(ssh_base_cmd + ["tmux","new","-d","-s","ollama", "ollama serve"], check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error occurred while setting up tmux session: {e}")
|
||||
# Pull the required Ollama model
|
||||
|
||||
Reference in New Issue
Block a user