Merge pull request #75 from agent-ecosystem/improve-base-url-error-handling

Improve OpenAI base url error handling

Author: dacharyc

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.5.6]
+
+### Fixed
+
+- Improve error message when the OpenAI API returns an `incorrect_hostname`
+  error due to regional endpoint requirements ([#70]). The error now tells
+  the user to set `OPENAI_BASE_URL` or `--base-url` with the correct
+  regional host (e.g. `https://us.api.openai.com/v1`).
+
 ## [1.5.5]
 
 ### Fixed
@@ -202,6 +211,7 @@ First stable release. Includes the complete CLI and importable library packages.
 - `types` — shared data types (`Report`, `Result`, `Level`, etc.)
 - `judge.LLMClient` interface for custom LLM providers
 
+[1.5.6]: https://github.com/agent-ecosystem/skill-validator/compare/v1.5.5...v1.5.6
 [1.5.5]: https://github.com/agent-ecosystem/skill-validator/compare/v1.5.4...v1.5.5
 [1.5.4]: https://github.com/agent-ecosystem/skill-validator/compare/v1.5.3...v1.5.4
 [1.5.3]: https://github.com/agent-ecosystem/skill-validator/compare/v1.5.2...v1.5.3

README.md

@@ -341,6 +341,12 @@ skill-validator score evaluate --provider claude-cli <path>
 
 \*\* The `openai` provider also reads these optional environment variables: `OPENAI_BASE_URL` (API base URL, overridden by `--base-url`), `OPENAI_ORG_ID` (sent as `OpenAI-Organization` header), and `OPENAI_PROJECT_ID` (sent as `OpenAI-Project` header). The org/project headers are only sent when targeting an OpenAI endpoint (e.g. `api.openai.com`, `us.api.openai.com`), not when using third-party compatible APIs.
 
+If your OpenAI organization is bound to a regional endpoint (e.g. `us.api.openai.com` or `eu.api.openai.com`), set `OPENAI_BASE_URL` to the full regional URL including the `/v1` path:
+
+```bash
+export OPENAI_BASE_URL=https://us.api.openai.com/v1
+```
+
 Use `--model` to override the default model and `--base-url` to point at any OpenAI-compatible endpoint (e.g. `http://localhost:11434/v1` for Ollama). If the endpoint requires a specific token limit parameter, use `--max-tokens-style` to override auto-detection:
 
 | Value | Behavior |

cmd/root.go

@@ -11,7 +11,7 @@ import (
 	"github.com/agent-ecosystem/skill-validator/types"
 )
 
-const version = "v1.5.5"
+const version = "v1.5.6"
 
 var (
 	outputFormat    string

judge/client.go

@@ -306,6 +306,16 @@ func (c *openaiClient) Complete(ctx context.Context, systemPrompt, userContent s
 	}
 
 	if resp.StatusCode != http.StatusOK {
+		var errResp struct {
+			Error struct {
+				Code    string `json:"code"`
+				Message string `json:"message"`
+			} `json:"error"`
+		}
+		if json.Unmarshal(respBody, &errResp) == nil && errResp.Error.Code == "incorrect_hostname" {
+			return "", fmt.Errorf("your OpenAI organization requires a regional endpoint: %s\nSet OPENAI_BASE_URL or use --base-url to specify the correct host (e.g. https://us.api.openai.com/v1)",
+				errResp.Error.Message)
+		}
 		return "", fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(respBody))
 	}
 

judge/client_test.go

@@ -269,6 +269,37 @@ func TestOpenAIClient_OrgProjectHeaders(t *testing.T) {
 	})
 }
 
+func TestOpenAIClient_RegionalHostnameError(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusUnauthorized)
+		_, _ = fmt.Fprint(w, `{"error":{"message":"Attempted to access resource with incorrect regional hostname. Please make your request to us.api.openai.com","type":"invalid_request_error","code":"incorrect_hostname","param":null},"status":401}`)
+	}))
+	defer server.Close()
+
+	client, err := NewClient(ClientOptions{
+		Provider: "openai",
+		APIKey:   "test-key",
+		BaseURL:  server.URL,
+		Model:    "gpt-4o",
+	})
+	if err != nil {
+		t.Fatalf("NewClient: %v", err)
+	}
+
+	_, err = client.Complete(t.Context(), "system", "user")
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+
+	if !strings.Contains(err.Error(), "regional endpoint") {
+		t.Errorf("expected error to mention regional endpoint, got: %v", err)
+	}
+	if !strings.Contains(err.Error(), "OPENAI_BASE_URL") {
+		t.Errorf("expected error to mention OPENAI_BASE_URL, got: %v", err)
+	}
+}
+
 // rewriteTransport rewrites requests to a different target URL while
 // preserving the original Host header for testing.
 type rewriteTransport struct {