1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187 | (ns ird.conformance-test
"Live spec-conformance sweep: call every read endpoint against
https://api.ird.ro and validate each response body against the schema
`spec/openapi.json` declares for it.
This is the complement to ird.coverage-test. That one asserts the method
and URL of all 161 operations against a capturing transport, which pins
the client to the spec but says nothing about whether the spec still
matches the API. This one closes that gap, and has caught three real
spec bugs: two endpoints returning fields their own
`additionalProperties: false` schema forbade, and an action-type enum
missing a value the API demonstrably returns.
Tagged :integration and skipped by default (see tests.edn); run with
clojure -M:test --focus-meta :integration
Read-only by design, like ird.integration-test: it creates nothing and
charges nothing beyond metered reads. Endpoints that need a resource to
exist are exercised against whatever the account already has and skipped
when it has none — the coverage note at the bottom of this namespace
lists what that leaves out and how to reach it by hand."
(:require
[cheshire.core :as json]
[clojure.java.io :as io]
[clojure.string :as str]
[clojure.test :refer [deftest is testing]]
[ird.client :as ird]
[ird.credits :as credits]
[ird.dns :as dns]
[ird.domains :as domains]
[ird.ipfs :as ipfs]
[ird.p2p :as p2p]
[ird.pins :as pins]
[ird.registrations :as reg]
[ird.sharing :as sharing]
[ird.spec-check :as spec-check]
[ird.support :as support]
[ird.transfers :as transfers]
[ird.tunnels :as tunnels]
[ird.test-env :as test-env]
[ird.vps :as vps]))
(defn- spec []
(json/parse-string (slurp (io/file "spec/openapi.json"))))
(defn- first-id
"The id of the first item under COLLECTION-KEY of (F client), or nil when
the account has none (or the listing is not readable with an API key)."
[client f collection-key]
(try (:id (first (get (f client) collection-key)))
(catch Exception _ nil)))
(defn- endpoints
"[spec-path expected-status thunk] for every read endpoint reachable with
this account's data. Thunks that need a resource the account lacks come
back nil and are dropped.
A few listings are account-JWT-only and answer 401 to an API key; those
carry 401 as their expected status, so their error envelope is validated
against the spec instead of their success body."
[client anon]
(let [id #(first-id client %1 %2)
reg-id (id reg/list-registrations :registrations)
gw-id (id ipfs/list-domains :domains)
name-id (id ipfs/list-names :names)
pin-id (id ipfs/list-pins :pins)
app-id (id p2p/list-apps :apps)
tunnel-id (id tunnels/list-tunnels :tunnels)
instance-id (id vps/list-instances :instances)
deployment-id (id vps/list-app-deployments :deployments)
action-id (id reg/list-actions :actions)
thread-id (id support/list-threads :threads)
claim-id (id dns/list-claims :claims)
zone (some-> (first (:domains (dns/list-zones client)))
:origin (str/replace #"\.$" ""))]
(->> [["/credits/balance" 200 #(credits/balance client)]
["/credits/ledger" 200 #(credits/ledger client)]
["/credits/prices" 200 #(credits/prices client)]
["/contact/threads" 200 #(support/list-threads client)]
["/contact/threads/{id}" 200 (when thread-id #(support/get-thread client thread-id))]
["/domains/check" 200 #(domains/check anon "ird.ro")]
["/domains/info" 200 #(domains/info client "ird.ro")]
["/domains/preflight" 200 #(domains/preflight client "ird.ro")]
["/domains/dns-snapshot" 200 #(domains/dns-snapshot client "ird.ro")]
["/domains/search" 200 #(domains/search client "shop")]
["/domains/example" 200 #(domains/example client)]
["/domains/directory" 200 #(domains/directory client {:limit 20})]
["/domains/directory/{domain}" 200 #(domains/directory-entry client "ird.ro")]
["/domains/registrations" 200 #(reg/list-registrations client {:limit 100})]
["/domains/registrations/{id}" 200 (when reg-id #(reg/get-registration client reg-id))]
["/domains/registrations/{id}/hosts" 200 (when reg-id #(reg/list-hosts client reg-id))]
["/domains/registrations/{id}/dsdata" 200 (when reg-id #(reg/list-dsdata client reg-id))]
["/domains/registrations/{id}/trade" 200 (when reg-id #(reg/quote-trade client reg-id))]
["/domains/registrations/{id}/dns/dnssec" 200 (when reg-id #(dns/get-dnssec client reg-id))]
["/domains/registrations/{id}/dns/nameservers" 200 (when reg-id #(dns/hosted-nameservers client reg-id))]
["/domains/actions" 200 #(reg/list-actions client {:limit 100})]
["/domains/actions/{id}" 200 (when action-id #(reg/get-action client action-id))]
["/dns/domains" 200 #(dns/list-zones client)]
["/dns/domains/{domain}" 200 (when zone #(dns/get-zone client zone))]
["/dns/claims" 200 #(dns/list-claims client)]
["/dns/claims/{id}" 200 (when claim-id #(dns/get-claim client claim-id))]
["/ipfs/pins" 200 #(ipfs/list-pins client {:limit 100})]
["/ipfs/pins/{id}" 200 (when pin-id #(ipfs/get-pin client pin-id))]
["/ipfs/names" 200 #(ipfs/list-names client)]
["/ipfs/names/{id}" 200 (when name-id #(ipfs/get-name client name-id))]
["/ipfs/domains" 200 #(ipfs/list-domains client)]
["/ipfs/domains/{id}" 200 (when gw-id #(ipfs/get-domain client gw-id))]
["/ipfs/domains/{id}/pins" 200 (when gw-id #(ipfs/list-domain-pins client gw-id))]
["/ipfs/usage" 200 #(ipfs/usage client)]
["/pins" 200 #(pins/list-pins client {:limit 100})]
["/p2p/apps" 200 #(p2p/list-apps client)]
["/p2p/apps/{id}" 200 (when app-id #(p2p/get-app client app-id))]
["/p2p/apps/{id}/domains" 200 (when app-id #(p2p/list-app-domains client app-id))]
["/p2p/usage" 200 #(p2p/usage client)]
["/tunnels" 200 #(tunnels/list-tunnels client)]
["/tunnels/{id}" 200 (when tunnel-id #(tunnels/get-tunnel client tunnel-id))]
["/vps/tiers" 200 #(vps/list-tiers client)]
["/vps/system-images" 200 #(vps/list-system-images client)]
["/vps/ssh-keys" 200 #(vps/list-ssh-keys client)]
["/vps/availability" 200 #(vps/availability client)]
["/vps/instances" 200 #(vps/list-instances client)]
["/vps/instances/{id}" 200 (when instance-id #(vps/get-instance client instance-id))]
["/vps/instances/{id}/metrics" 200 (when instance-id #(vps/get-metrics client instance-id))]
["/vps/instances/{id}/domains" 200 (when instance-id #(vps/list-instance-domains client instance-id))]
["/vps/apps" 200 #(vps/app-catalog client)]
["/vps/apps/deployments" 200 #(vps/list-app-deployments client)]
["/vps/apps/deployments/{id}" 200 (when deployment-id #(vps/get-app-deployment client deployment-id))]
["/vps/volumes" 200 #(vps/list-volumes client)]
["/vps/images" 200 #(vps/list-images client)]
["/vps/activity" 200 #(vps/list-activity client)]
;; account-JWT-only: the 401 envelope is the documented response here
["/grants" 401 #(sharing/list-grants client)]
["/domains/registrations/{id}/grants" 401 (when reg-id #(sharing/list-domain-grants client reg-id))]
["/vps/transfers" 401 #(transfers/list-transfers client)]]
(remove (comp nil? #(nth % 2)))
vec)))
(defn- sweep
"Call each endpoint and validate its body, returning
[{:path :status :problems}]."
[spec client anon]
(for [[path expected thunk] (endpoints client anon)]
(let [result (try {:body (thunk)}
(catch clojure.lang.ExceptionInfo e
{:error e :status (:status (ex-data e))}))
status (or (:status result) 200)
body (or (:body result) (:response (ex-data (:error result))))]
{:path path
:status status
:expected expected
:problems (if (= status expected)
(spec-check/check (spec-check/response-schema spec :get path status)
body "")
[(str "expected HTTP " expected " but got " status)])})))
(deftest ^:integration live-responses-match-the-spec
(if-let [key (test-env/api-key)]
(let [spec (spec)
results (sweep spec (ird/make-client {:api-key key}) (ird/make-client {}))
drifted (remove (comp empty? :problems) results)]
(doseq [{:keys [path problems]} drifted]
(println "\nDRIFT" path)
(doseq [p problems] (println " -" p)))
(println (format "\nconformance: %d endpoints, %d conform, %d drifted"
(count results) (- (count results) (count drifted))
(count drifted)))
(testing "every reachable read endpoint returns what the spec declares"
(is (seq results) "no endpoints were exercised")
(is (empty? drifted)
(str "spec drift on " (mapv :path drifted)
" — the spec and the live API disagree; see the report above"))))
(println "IRD_API_KEY not set - skipping the conformance sweep")))
;; Not covered here, and why. Each needs a resource that only exists mid-flight
;; or costs money to create, so reaching it means a deliberate manual run
;; rather than something a test suite should do on its own:
;;
;; GET /jobs/{id}, /jobs/{id}/result enqueue a job (jobs/run)
;; GET /ipfs/uploads/{id} start an upload (ipfs/add-file)
;; GET /pins/{requestid} add a pin through ird.pins
;; GET /vps/actions/{id} any VPS action; create a server
;;
;; The four VPS instance endpoints above DO run whenever the account has a
;; server. All four, plus /vps/actions/{id}, were validated by hand against a
;; throwaway instance; the others by hand against throwaway jobs, uploads and
;; pins. None of the three spec bugs found so far needed any of that — every
;; one surfaced on a plain read endpoint that this sweep covers.
|