Login an Firmware 010146.5.0.001.0 anpassen

Die bisherige Anmeldung gehörte zu einer anderen Firmware-Generation und
scheiterte auf diesem Gerät durchgängig mit "login: failed". Der Ablauf wurde
aus js/pages/login.js und js/jquery-addons.js des Routers rekonstruiert:

1. Challenge per CCM-verschlüsseltem POST {getChallenge:1, httoken}
   an data/Login.json; das Antwortfeld heisst "challenge", nicht "challengev".
2. Passwort = SHA256(challenge + ":" + passwort), hex — kein PBKDF2/AES-CCM.
3. Nach dem Login dient die Challenge selbst als CCM-Schluessel
   (im Client getFromLocal("challenge_val")), nicht mehr der statische.
4. _httoken steht als `var _httoken = <zahl>` inline in angemeldeten Seiten
   und wird nach dem Login von dort gelesen.
5. Datenabrufe brauchen Cache-Buster, _tn und den Header
   X-Requested-With: XMLHttpRequest — ohne ihn liefert der Router die
   HTML-Huelle statt Nutzdaten.

Ausserdem: Login() wurde bisher nirgends aufgerufen (toter Code), und
login_locked wird jetzt als eigene Fehlermeldung ausgegeben. Es ist ein
exponentieller Backoff (2 -> 4 -> 16 -> 32) und die Folge fehlgeschlagener
Versuche, nicht deren Ursache.

Ergebnis: acht Endpunkte liefern Daten. data/DSL.json existiert auf dieser
Firmware nicht (404). data/PortuwMain.json und data/LAN.json antworten
weiterhin mit status=ok ohne Nutzlast; die fuenf geprueften Anfrageformen
stehen als Kommentar an der betreffenden Stelle.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

internal/speedport/client.go

@@ -11,13 +11,16 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
+	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
-
-	"golang.org/x/crypto/pbkdf2"
 )
 
+// httokenRe matches `var _httoken = 1931431431` in authenticated pages.
+var httokenRe = regexp.MustCompile(`var\s+_httoken\s*=\s*"?([A-Za-z0-9]+)"?`)
+
 const (
 	// Hardcoded AES-256-CCM key used by the Speedport Smart 4 for Status.json encryption.
 	// This was reverse-engineered by the community (see: github.com/aaronk6/dsl-monitoring).
@@ -49,6 +52,14 @@ type Client struct {
 	mu        sync.Mutex
 	csrfToken string
 	loggedIn  bool
+	// httoken is a form token the router's UI injects into its pages. The
+	// login flow sends it along; empty works for the challenge request.
+	httoken string
+	// After login the router switches to the challenge as CCM key for
+	// protected endpoints — mirroring getFromLocal("challenge_val") in its
+	// js/jquery-addons.js. Unprotected endpoints keep using the static key.
+	sessionKey []byte
+	sessionCCM *CCM
 
 	logger *slog.Logger
 }
@@ -114,14 +125,46 @@ func NewClient(host string, opts ...ClientOption) (*Client, error) {
 }
 
 // fetchRaw fetches a URL and returns the raw response body.
-func (c *Client) fetchRaw(path string) ([]byte, error) {
+// fetchPlain GETs a path verbatim, without the AJAX decorations.
-	reqURL := c.baseURL + "/" + path
+func (c *Client) fetchPlain(path string) ([]byte, error) {
+	req, err := http.NewRequest("GET", c.baseURL+"/"+path, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept-Language", "de-DE,de;q=0.9")
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	return io.ReadAll(resp.Body)
+}
 
-	req, err := http.NewRequest("GET", reqURL, nil)
+func (c *Client) fetchRaw(path string) ([]byte, error) {
+	// The UI appends cache busters and the session token as _tn (see
+	// addTokenField() in js/global.js) and marks the call as XHR. Without the
+	// marker the firmware answers with the HTML shell instead of payload.
+	referer := c.baseURL + "/html/content/overview/index.html"
+	if c.httoken != "" && !strings.Contains(path, "?") {
+		var tok string
+		tok, referer = c.tokenFor(path)
+		now := time.Now()
+		q := url.Values{
+			"_time": {strconv.FormatInt(now.UnixMilli(), 10)},
+			"_rand": {strconv.FormatInt(now.UnixNano()%1000, 10)},
+			"_lang": {"DE"},
+			"_tn":   {tok},
+		}
+		path = path + "?" + q.Encode()
+	}
+
+	req, err := http.NewRequest("GET", c.baseURL+"/"+path, nil)
 	if err != nil {
 		return nil, fmt.Errorf("create request: %w", err)
 	}
-	req.Header.Set("Accept-Language", "en")
+	req.Header.Set("Accept-Language", "de-DE,de;q=0.9")
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	req.Header.Set("Referer", referer)
 
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
@@ -133,15 +176,37 @@ func (c *Client) fetchRaw(path string) ([]byte, error) {
 		return nil, fmt.Errorf("HTTP GET %s: status %d", path, resp.StatusCode)
 	}
 
-	body, err := io.ReadAll(resp.Body)
+	return io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("read response: %w", err)
-	}
-
-	return body, nil
 }
 
 // postForm sends a POST request with form data.
+// postBody posts a pre-encoded body (the CCM ciphertext, hex) to path.
+func (c *Client) postBody(path, body string) ([]byte, error) {
+	req, err := http.NewRequest("POST", c.baseURL+"/"+path, strings.NewReader(body))
+	if err != nil {
+		return nil, fmt.Errorf("create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("Accept-Language", "de-DE,de;q=0.9")
+	// The firmware serves the HTML login page instead of encrypted payload
+	// unless the request looks like the UI's own AJAX call.
+	req.Header.Set("X-Requested-With", "XMLHttpRequest")
+	req.Header.Set("Origin", c.baseURL)
+	req.Header.Set("Referer", c.baseURL+"/html/content/overview/index.html")
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("HTTP POST %s: %w", path, err)
+	}
+	defer resp.Body.Close()
+
+	out, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response: %w", err)
+	}
+	return out, nil
+}
+
 func (c *Client) postForm(path string, data url.Values) ([]byte, error) {
 	reqURL := c.baseURL + "/" + path
 
@@ -178,11 +243,16 @@ func (c *Client) decryptStatusResponse(hexData []byte) ([]byte, error) {
 		return nil, fmt.Errorf("hex decode: %w", err)
 	}
 
-	// Nonce is the first 8 bytes of the key
+	// Once logged in the router encrypts protected endpoints with the
-	nonce := c.statusKey[:8]
+	// challenge as key; unprotected ones (Status.json, DeviceList.json) stay
+	// on the static key. Try the session key first, then fall back.
+	if c.sessionCCM != nil {
+		if plaintext, err := c.sessionCCM.Decrypt(c.sessionKey[:8], ciphertextWithTag, nil); err == nil {
+			return plaintext, nil
+		}
+	}
 
-	// Decrypt using AES-256-CCM
+	plaintext, err := c.statusCCM.Decrypt(c.statusKey[:8], ciphertextWithTag, nil)
-	plaintext, err := c.statusCCM.Decrypt(nonce, ciphertextWithTag, nil)
 	if err != nil {
 		return nil, fmt.Errorf("CCM decrypt: %w", err)
 	}
@@ -231,8 +301,19 @@ func (c *Client) GetStatus() (*StatusData, error) {
 	return NewStatusData(entries), nil
 }
 
-// Login authenticates with the router using the challenge-response mechanism.
+// Login authenticates with the router.
-// This is required for accessing protected endpoints like IPData.json or devices list.
+//
+// Flow as implemented by the router's own js/pages/login.js on firmware
+// 010146.5.0.001.0:
+//
+//  1. POST a CCM-encrypted body {getChallenge:1, httoken:…} to data/Login.json
+//     and decrypt the reply; the challenge is in the field "challenge".
+//  2. password = SHA256(challenge + ":" + plaintext password), hex.
+//  3. POST challenge + hashed password + showpw=0.
+//
+// An earlier implementation used PBKDF2 + AES-128-CCM and looked for a field
+// named "challengev". That belongs to a different firmware generation; this
+// router answers such requests with "login: failed" and raises login_locked.
 func (c *Client) Login() error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -243,115 +324,122 @@ func (c *Client) Login() error {
 
 	c.logger.Debug("starting login")
 
-	// Step 1: Request challenge
+	// Step 1: request the challenge.
-	challengeData, err := c.postForm("data/Login.json", url.Values{
+	challengeSD, err := c.postEncrypted(url.Values{
-		"csrf_token": {"nulltoken"},
+		"getChallenge": {"1"},
-		"showpw":     {"0"},
+		"httoken":      {c.httoken},
-		"password":   {c.hashPassword()},
 	})
 	if err != nil {
 		return fmt.Errorf("request challenge: %w", err)
 	}
 
-	// The challenge response might be encrypted too
+	challenge := challengeSD.Get("challenge")
-	var challengeEntries []VarEntry
+	if challenge == "" {
-	if err := json.Unmarshal(challengeData, &challengeEntries); err != nil {
-		// Try decrypting
-		plaintext, err2 := c.decryptStatusResponse(challengeData)
-		if err2 != nil {
-			return fmt.Errorf("parse challenge (raw parse: %v, decrypt: %v)", err, err2)
-		}
-		if err := json.Unmarshal(plaintext, &challengeEntries); err != nil {
-			return fmt.Errorf("parse decrypted challenge: %w", err)
-		}
-	}
-
-	challengeSD := NewStatusData(challengeEntries)
-
-	// Extract challenge value
-	challengeV := challengeSD.Get("challengev")
-	if challengeV == "" {
-		// Check if already logged in
 		if challengeSD.Get("login") == "success" {
 			c.loggedIn = true
 			c.csrfToken = challengeSD.Get("csrf_token")
-			c.logger.Info("login successful (no challenge needed)")
+			c.logger.Info("already logged in")
 			return nil
 		}
-		return fmt.Errorf("no challengev in response, status: %s", challengeSD.Get("login"))
+		// login_locked is an exponential backoff the router raises on every
+		// failed attempt and decays over time. Observed: 2 -> 4 -> 16 -> 32.
+		// It is a consequence of failed logins, not an independent lockout.
+		if locked := challengeSD.Get("login_locked"); locked != "" && locked != "0" {
+			return fmt.Errorf("no challenge returned; router backoff active "+
+				"(login_locked=%s) — wait for it to decay before retrying", locked)
+		}
+		return fmt.Errorf("no challenge in response, status: %s", challengeSD.Get("login"))
 	}
 
-	c.logger.Debug("got challenge", "length", len(challengeV))
+	c.logger.Debug("got challenge", "length", len(challenge))
 
-	// Step 2: Derive key from password + challenge
+	// Step 2: SHA256(challenge + ":" + password)
-	sha256Pwd := c.hashPassword()
+	sum := sha256.Sum256([]byte(challenge + ":" + c.password))
+	hashed := hex.EncodeToString(sum[:])
 
-	// PBKDF2: password=SHA256(device_password), salt=challengev[:16] as bytes
+	// Step 3: submit the login.
-	salt := []byte(challengeV[:16])
+	loginSD, err := c.postEncrypted(url.Values{
-	derivedKey := pbkdf2.Key([]byte(sha256Pwd), salt, pbkdf2Iterations, pbkdf2KeyLen, sha256.New)
+		"challenge": {challenge},
-
+		"password":  {hashed},
-	// Step 3: Encrypt the hashed password with AES-128-CCM
+		"showpw":    {"0"},
-	loginCCM, err := NewCCM(derivedKey, loginTagSize)
+		"httoken":   {c.httoken},
-	if err != nil {
-		return fmt.Errorf("create login CCM: %w", err)
-	}
-
-	// Nonce = hex decode of challengev[16:32]
-	nonce, err := hex.DecodeString(challengeV[16:32])
-	if err != nil {
-		return fmt.Errorf("decode nonce from challenge: %w", err)
-	}
-
-	// AAD = hex decode of challengev[32:48]
-	aad, err := hex.DecodeString(challengeV[32:48])
-	if err != nil {
-		return fmt.Errorf("decode AAD from challenge: %w", err)
-	}
-
-	// Encrypt the SHA256 password hash
-	encryptedPwd, err := loginCCM.Encrypt(nonce, []byte(sha256Pwd), aad)
-	if err != nil {
-		return fmt.Errorf("encrypt password: %w", err)
-	}
-
-	encryptedPwdHex := hex.EncodeToString(encryptedPwd)
-
-	// Step 4: Send login request with encrypted password
-	loginData, err := c.postForm("data/Login.json", url.Values{
-		"csrf_token": {"nulltoken"},
-		"showpw":     {"0"},
-		"password":   {encryptedPwdHex},
-		"challengev": {challengeV},
 	})
 	if err != nil {
 		return fmt.Errorf("login request: %w", err)
 	}
 
-	// Parse login response
-	var loginEntries []VarEntry
-	if err := json.Unmarshal(loginData, &loginEntries); err != nil {
-		plaintext, err2 := c.decryptStatusResponse(loginData)
-		if err2 != nil {
-			return fmt.Errorf("parse login response: %w (decrypt: %v)", err, err2)
-		}
-		if err := json.Unmarshal(plaintext, &loginEntries); err != nil {
-			return fmt.Errorf("parse decrypted login response: %w", err)
-		}
-	}
-
-	loginSD := NewStatusData(loginEntries)
-
 	if loginSD.Get("login") != "success" {
+		if locked := loginSD.Get("login_locked"); locked != "" && locked != "0" {
+			return fmt.Errorf("login rejected (login_locked=%s)", locked)
+		}
 		return fmt.Errorf("login failed: %s", loginSD.Get("login"))
 	}
 
 	c.csrfToken = loginSD.Get("csrf_token")
 	c.loggedIn = true
+
+	if key, err := hex.DecodeString(challenge); err == nil && len(key) == 32 {
+		if sc, err := NewCCM(key, statusTagSize); err == nil {
+			c.sessionKey = key
+			c.sessionCCM = sc
+			c.logger.Debug("session key derived from challenge")
+		}
+	}
+
+	c.fetchHTToken()
+
 	c.logger.Info("login successful")
 
 	return nil
 }
 
+// fetchHTToken retrieves the session token the router injects inline into its
+// authenticated pages as `var _httoken = <number>`. Its own UI reads that
+// variable and appends it to every data request (setHTToken in js/global.js).
+func (c *Client) fetchHTToken() {
+	page, err := c.fetchRaw("html/content/overview/index.html")
+	if err != nil {
+		c.logger.Debug("httoken: page fetch failed", "error", err)
+		return
+	}
+	if m := httokenRe.FindSubmatch(page); m != nil {
+		c.httoken = string(m[1])
+		c.logger.Debug("httoken acquired", "length", len(c.httoken))
+		return
+	}
+	c.logger.Debug("httoken: not found in page")
+}
+
+// postEncrypted CCM-encrypts a form body, posts it to data/Login.json and
+// decrypts the reply. Mirrors encryptccm()/decryptccm() in the router's
+// js/jquery-addons.js: the static Status.json key is used while no challenge
+// is stored client-side, the nonce is the first 8 bytes of that key, and the
+// wire format is hex.
+func (c *Client) postEncrypted(form url.Values) (*StatusData, error) {
+	ct, err := c.statusCCM.Encrypt(c.statusKey[:8], []byte(form.Encode()), nil)
+	if err != nil {
+		return nil, fmt.Errorf("encrypt request: %w", err)
+	}
+
+	raw, err := c.postBody("data/Login.json", hex.EncodeToString(ct))
+	if err != nil {
+		return nil, err
+	}
+
+	var entries []VarEntry
+	if err := json.Unmarshal(raw, &entries); err != nil {
+		plaintext, derr := c.decryptStatusResponse(raw)
+		if derr != nil {
+			return nil, fmt.Errorf("parse reply (raw: %v, decrypt: %v)", err, derr)
+		}
+		if err := json.Unmarshal(plaintext, &entries); err != nil {
+			return nil, fmt.Errorf("parse decrypted reply: %w", err)
+		}
+	}
+
+	return NewStatusData(entries), nil
+}
+
 // hashPassword returns the SHA256 hex digest of the password.
 func (c *Client) hashPassword() string {
 	h := sha256.Sum256([]byte(c.password))
@@ -362,25 +450,67 @@ func (c *Client) hashPassword() string {
 func (c *Client) GetEndpoint(endpoint string) (*StatusData, error) {
 	c.logger.Debug("fetching endpoint", "endpoint", endpoint)
 
+	// Two request forms exist and no endpoint accepts both:
+	//   POST — how the UI asks ($.postJSON in js/jquery-addons.js): cache-busting
+	//          query parameters, an XMLHttpRequest marker header and a
+	//          CCM-encrypted body carrying httoken. Required for IPData, InetIP,
+	//          Connect, WLANBasic and SystemMessages.
+	//   GET  — plain, works for Status, DeviceList, Modules and Overview.
+	// Try the authenticated POST first, fall back to GET.
+	if c.httoken != "" {
+		if sd, err := c.fetchViaPost(endpoint); err == nil {
+			return sd, nil
+		} else {
+			c.logger.Debug("POST form failed, falling back to GET", "endpoint", endpoint, "error", err)
+		}
+	}
+
 	raw, err := c.fetchRaw(endpoint)
 	if err != nil {
 		return nil, fmt.Errorf("fetch %s: %w", endpoint, err)
 	}
+	return c.parsePayload(endpoint, raw)
+}
 
-	// Try direct JSON parse
+// fetchViaPost issues the authenticated POST the router's own UI uses.
+func (c *Client) fetchViaPost(endpoint string) (*StatusData, error) {
+	key, ccm := c.statusKey, c.statusCCM
+	if c.sessionCCM != nil {
+		key, ccm = c.sessionKey, c.sessionCCM
+	}
+
+	ct, err := ccm.Encrypt(key[:8], []byte(url.Values{"httoken": {c.httoken}}.Encode()), nil)
+	if err != nil {
+		return nil, fmt.Errorf("encrypt request: %w", err)
+	}
+
+	now := time.Now()
+	q := url.Values{
+		"_time": {strconv.FormatInt(now.UnixMilli(), 10)},
+		"_rand": {strconv.FormatInt(now.UnixNano()%1000, 10)},
+		"_lang": {"DE"},
+	}
+
+	raw, err := c.postBody(endpoint+"?"+q.Encode(), hex.EncodeToString(ct))
+	if err != nil {
+		return nil, err
+	}
+	return c.parsePayload(endpoint, raw)
+}
+
+// parsePayload turns a raw reply into StatusData, decrypting when needed.
+func (c *Client) parsePayload(endpoint string, raw []byte) (*StatusData, error) {
 	var entries []VarEntry
 	if err := json.Unmarshal(raw, &entries); err == nil {
 		return NewStatusData(entries), nil
 	}
 
-	// Try decrypting
 	plaintext, err := c.decryptStatusResponse(raw)
 	if err != nil {
 		return nil, fmt.Errorf("decrypt %s: %w", endpoint, err)
 	}
 
-	entries, err = parseResponse(plaintext)
+	if err := json.Unmarshal(plaintext, &entries); err != nil {
-	if err != nil {
 		return nil, fmt.Errorf("parse %s: %w", endpoint, err)
 	}
 
@@ -445,6 +575,42 @@ func (c *Client) IsLoggedIn() bool {
 }
 
 // AllEndpoints is a list of known Speedport Smart 4 data endpoints.
+// endpointPages maps a data endpoint to the UI page that requests it. The
+// router hands out a fresh _tn per page, and a token from a different page is
+// accepted but yields an empty payload — so the page must be loaded first.
+// NOTE: data/PortuwMain.json and data/LAN.json answer with only {status: ok}
+// for this client, although the UI shows eleven port forwards and an active
+// DHCP range. Tried without success: plain GET, GET with _tn from the overview
+// page, GET with a page-specific _tn plus matching Referer, the UI's exact
+// query string (duplicated _time/_rand, _tn last), and the encrypted POST form.
+// All other authenticated endpoints work with the plain GET below, so the
+// session itself is fine. Whatever these two additionally require has not been
+// identified; their contents were taken from the web UI by hand.
+var endpointPages = map[string]string{
+	"data/PortuwMain.json": "html/content/internet/portforwarding.html",
+	"data/LAN.json":        "html/content/network/dhcp.html",
+	"data/DeviceList.json": "html/content/network/devices.html",
+}
+
+// tokenFor returns the _tn belonging to the page that owns endpoint, falling
+// back to the session-wide token from the overview page.
+func (c *Client) tokenFor(endpoint string) (token, referer string) {
+	page, ok := endpointPages[endpoint]
+	if !ok {
+		return c.httoken, c.baseURL + "/html/content/overview/index.html"
+	}
+	referer = c.baseURL + "/" + page + "?lang=de"
+	body, err := c.fetchPlain(page)
+	if err != nil {
+		c.logger.Debug("page token: fetch failed", "page", page, "error", err)
+		return c.httoken, referer
+	}
+	if m := httokenRe.FindSubmatch(body); m != nil {
+		return string(m[1]), referer
+	}
+	return c.httoken, referer
+}
+
 var AllEndpoints = []string{
 	"data/Status.json",
 	"data/DSL.json",
@@ -456,4 +622,8 @@ var AllEndpoints = []string{
 	"data/WLANBasic.json",
 	"data/Overview.json",
 	"data/SystemMessages.json",
+	// Found by watching the router's own UI: port forwarding and the LAN/DHCP
+	// page request these two.
+	"data/PortuwMain.json",
+	"data/LAN.json",
 }

main.go

@@ -73,6 +73,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Authenticate when a password was supplied. Without this, the protected
+	// endpoints (IPData, InetIP, Connect, WLANBasic, ...) return the HTML login
+	// page instead of encrypted payload, which surfaces as a hex decode error.
+	// Status.json works either way and stays available if the login fails.
+	if *password != "" {
+		if err := client.Login(); err != nil {
+			logger.Warn("login failed — protected endpoints will be unavailable", "error", err)
+		} else {
+			logger.Debug("login succeeded")
+		}
+	}
+
 	switch subcommand {
 	case "status":
 		cmdStatus(client, logger)