feat(lib-provider): add deutsche wohnen provider

README.md

@@ -135,7 +135,7 @@ Fredy is built around three simple concepts:
 ### Provider 🌐
 
 A **provider** is a real-estate platform (e.g. ImmoScout24, Immowelt,
-Immonet, eBay Kleinanzeigen, WG-Gesucht).\
+Immonet, Deutsche Wohnen, eBay Kleinanzeigen, WG-Gesucht).\
 When you create a job, you paste the search URL from the platform into
 Fredy.\
 ⚠️ Always make sure the search results are sorted by **date**, so Fredy

lib/provider/deutscheWohnen.js

@@ -0,0 +1,258 @@
+/*
+ * Copyright (c) 2026 by Christian Kellner.
+ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
+ */
+
+/**
+ * Deutsche Wohnen provider using the site's JSON API to retrieve listings.
+ *
+ * Users paste a search URL from https://www.deutsche-wohnen.com/mieten/mietangebote
+ * which is translated to the internal API endpoint:
+ * GET /api/deuwo-real-estate/list?{search parameters}
+ *
+ * Detail pages are server-rendered and embed structured listing data in the
+ * `data-vonovia-data` attribute, which is parsed during fetchDetails().
+ */
+
+import { buildHash, isOneOf } from '../utils.js';
+import logger from '../services/logger.js';
+import { extractNumber } from '../utils/extract-number.js';
+import puppeteerExtractor from '../services/extractor/puppeteerExtractor.js';
+import * as cheerio from 'cheerio';
+/** @import { ParsedListing } from '../types/listing.js' */
+/** @import { ProviderConfig } from '../types/providerConfig.js' */
+
+/** @type {string[]} */
+let appliedBlackList = [];
+/** @type {string | null} */
+let refererUrl = null;
+
+/**
+ * Translates a Deutsche Wohnen search page URL into the JSON API endpoint.
+ *
+ * @param {string} url Web search URL or API URL pasted by the user.
+ * @returns {string} API URL used by getListings().
+ */
+export function convertWebToApi(url) {
+  const parsed = new URL(url);
+
+  if (parsed.pathname === '/api/deuwo-real-estate/list') {
+    return parsed.toString();
+  }
+
+  const params = parsed.searchParams;
+  params.delete('scroll');
+  params.set('limit', params.get('limit') || '100');
+  params.set('dataSet', params.get('dataSet') || 'deuwo');
+
+  for (const key of ['parkingCarport', 'parkingStellplatz', 'parkingGarage', 'parkingTiefgarage']) {
+    if (!params.has(key)) {
+      params.set(key, '0');
+    }
+  }
+
+  return `${metaInformation.baseUrl}api/deuwo-real-estate/list?${params.toString()}`;
+}
+
+/**
+ * @param {any} item
+ * @returns {string}
+ */
+function buildAddress(item) {
+  const street = item.strasse?.trim();
+  const cityLine = [item.plz, item.ort].filter(Boolean).join(' ').trim();
+  return [street, cityLine].filter(Boolean).join(', ');
+}
+
+/**
+ * @param {number | null | undefined} value
+ * @returns {number | null}
+ */
+function normalizeCoordinate(value) {
+  if (value == null || value === 0) {
+    return null;
+  }
+  return value;
+}
+
+/**
+ * @param {any} detailData
+ * @returns {string}
+ */
+function buildDescription(detailData) {
+  const parts = [];
+
+  if (detailData.description?.trim()) {
+    parts.push(`Beschreibung\n${detailData.description.trim()}`);
+  }
+
+  if (detailData.features?.length) {
+    parts.push(
+      `Ausstattung\n${detailData.features
+        .map((f) => f.trim())
+        .filter(Boolean)
+        .join('\n')}`,
+    );
+  }
+
+  if (detailData.location?.trim()) {
+    parts.push(`Lage\n${detailData.location.trim()}`);
+  }
+
+  const sectionText = (detailData.sections || [])
+    .map((section) => {
+      const rows = (section.rows || [])
+        .filter((row) => row.label && row.value)
+        .map((row) => `${row.label}: ${row.value}`)
+        .join('\n');
+      if (!rows) return null;
+      return [section.heading, rows].filter(Boolean).join('\n');
+    })
+    .filter(Boolean)
+    .join('\n\n');
+
+  if (sectionText) {
+    parts.push(sectionText);
+  }
+
+  if (detailData.miscellaneous?.trim()) {
+    parts.push(detailData.miscellaneous.trim());
+  }
+
+  return parts.join('\n\n').trim();
+}
+
+/**
+ * @param {string} url
+ * @returns {Promise<any[]>}
+ */
+async function getListings(url) {
+  const response = await fetch(url, {
+    method: 'GET',
+    headers: {
+      'User-Agent':
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36',
+      Accept: 'application/json',
+      ...(refererUrl ? { Referer: refererUrl } : {}),
+    },
+  });
+
+  if (!response.ok) {
+    logger.error('Error fetching data from Deutsche Wohnen API:', response.statusText);
+    return [];
+  }
+
+  const responseBody = await response.json();
+  return (responseBody.results || [])
+    .filter((item) => item.vermarktungsart_miete === '1')
+    .map((item) => ({
+      id: item.wrk_id,
+      price: item.preis,
+      size: item.groesse,
+      rooms: item.anzahl_zimmer,
+      title: item.titel,
+      link: `${metaInformation.baseUrl}mieten/mietangebote/${item.slug}`,
+      address: buildAddress(item),
+      image: item.preview_img_url,
+      latitude: item.lat,
+      longitude: item.lng,
+    }));
+}
+
+/**
+ * @param {ParsedListing} listing
+ * @param {any} [browser]
+ * @returns {Promise<ParsedListing>}
+ */
+async function fetchDetails(listing, browser) {
+  try {
+    const html = await puppeteerExtractor(listing.link, 'body', { browser, name: 'deutscheWohnen_details' });
+    if (!html) return listing;
+
+    const $ = cheerio.load(html);
+    const rawData = $('[data-vonovia-data]').attr('data-vonovia-data');
+    if (!rawData) return listing;
+
+    const detailData = JSON.parse(rawData);
+    const description = buildDescription(detailData);
+    const address = [detailData.streetAndHouseNumber, detailData.postCodeAndCity].filter(Boolean).join(', ');
+
+    return {
+      ...listing,
+      address: address || listing.address,
+      description: description || listing.description,
+      latitude: normalizeCoordinate(detailData.latitude) ?? listing.latitude,
+      longitude: normalizeCoordinate(detailData.longitude) ?? listing.longitude,
+    };
+  } catch (error) {
+    logger.warn(`Could not fetch Deutsche Wohnen detail page for listing '${listing.id}'.`, error?.message || error);
+    return listing;
+  }
+}
+
+/**
+ * @param {any} o
+ * @returns {ParsedListing}
+ */
+function normalize(o) {
+  const id = buildHash(o.id, o.price);
+  const price = extractNumber(o.price);
+  return {
+    id,
+    link: o.link,
+    title: (o.title || '').trim(),
+    price: price != null ? Math.round(price) : null,
+    size: extractNumber(o.size),
+    rooms: extractNumber(o.rooms),
+    address: o.address,
+    image: o.image,
+    description: o.description,
+    latitude: normalizeCoordinate(o.latitude),
+    longitude: normalizeCoordinate(o.longitude),
+  };
+}
+
+/**
+ * @param {ParsedListing} o
+ * @returns {boolean}
+ */
+function applyBlacklist(o) {
+  const titleNotBlacklisted = !isOneOf(o.title, appliedBlackList);
+  const descNotBlacklisted = !isOneOf(o.description, appliedBlackList);
+  return titleNotBlacklisted && descNotBlacklisted;
+}
+
+/** @type {ProviderConfig} */
+const config = {
+  requiredFieldNames: ['id', 'link', 'title', 'price', 'size', 'rooms', 'address', 'image', 'description'],
+  url: null,
+  crawlFields: {
+    id: 'wrk_id',
+    title: 'titel',
+    price: 'preis',
+    size: 'groesse',
+    rooms: 'anzahl_zimmer',
+    link: 'slug',
+    address: 'strasse',
+    image: 'preview_img_url',
+  },
+  normalize,
+  filter: applyBlacklist,
+  getListings,
+  fetchDetails,
+};
+
+export const init = (sourceConfig, blacklist) => {
+  config.enabled = sourceConfig.enabled;
+  refererUrl = sourceConfig.url;
+  config.url = convertWebToApi(sourceConfig.url);
+  appliedBlackList = blacklist || [];
+};
+
+export const metaInformation = {
+  name: 'Deutsche Wohnen',
+  baseUrl: 'https://www.deutsche-wohnen.com/',
+  id: 'deutscheWohnen',
+};
+
+export { config };

test/offlineFixtures.js

@@ -88,6 +88,7 @@ export async function readFixture(url) {
 export function buildFetchMock() {
   let listData = null;
   let detailData = null;
+  let deutscheWohnenListData = null;
 
   return async (url) => {
     const urlStr = String(url);
@@ -111,6 +112,14 @@ export function buildFetchMock() {
       return { ok: true, status: 200, json: () => Promise.resolve(detailData) };
     }
 
+    if (urlStr.includes('deutsche-wohnen.com/api/deuwo-real-estate/list')) {
+      if (!deutscheWohnenListData) {
+        const raw = await tryReadFile(path.join(FIXTURES_DIR, 'deutscheWohnen_list.json'));
+        deutscheWohnenListData = raw ? JSON.parse(raw) : { results: [] };
+      }
+      return { ok: true, status: 200, json: () => Promise.resolve(deutscheWohnenListData) };
+    }
+
     throw new Error(`Network request blocked in offline mode: ${urlStr}`);
   };
 }

test/provider/deutscheWohnen.test.js

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2026 by Christian Kellner.
+ * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause
+ */
+
+import { expect } from 'vitest';
+import * as similarityCache from '../../lib/services/similarity-check/similarityCache.js';
+import { mockFredy, providerConfig } from '../utils.js';
+import { get } from '../mocks/mockNotification.js';
+import * as provider from '../../lib/provider/deutscheWohnen.js';
+
+// Deutsche Wohnen uses a JSON API (fetch-based, no browser). Both tests share
+// the same module-level listings so the API is only queried once.
+const TEST_TIMEOUT = 120_000;
+
+describe('#deutscheWohnen provider testsuite()', () => {
+  provider.init(providerConfig.deutscheWohnen, [], []);
+
+  let liveListings;
+
+  it(
+    'should test deutscheWohnen provider',
+    async () => {
+      const Fredy = await mockFredy();
+      const mockedJob = {
+        id: 'deutscheWohnen',
+        notificationAdapter: null,
+        spatialFilter: null,
+        specFilter: null,
+      };
+
+      const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache, undefined);
+
+      liveListings = await fredy.execute();
+
+      if (liveListings == null || liveListings.length === 0) {
+        throw new Error('Listings is empty!');
+      }
+
+      expect(liveListings).toBeInstanceOf(Array);
+      const notificationObj = get();
+      expect(notificationObj).toBeTypeOf('object');
+      expect(notificationObj.serviceName).toBe('deutscheWohnen');
+
+      const hasValidNotification = notificationObj.payload.some((notify) => {
+        return (
+          typeof notify.id === 'string' &&
+          typeof notify.price === 'string' &&
+          notify.price.includes('€') &&
+          typeof notify.size === 'string' &&
+          notify.size.includes('m²') &&
+          typeof notify.title === 'string' &&
+          notify.title !== '' &&
+          typeof notify.link === 'string' &&
+          notify.link.includes('https://www.deutsche-wohnen.com/') &&
+          typeof notify.address === 'string' &&
+          notify.address !== ''
+        );
+      });
+
+      expect(hasValidNotification).toBe(true);
+    },
+    TEST_TIMEOUT,
+  );
+
+  describe('with provider_details enabled', () => {
+    it(
+      'should enrich listings with details',
+      async () => {
+        if (!liveListings?.length) throw new Error('No listings from first test to enrich');
+
+        const enriched = await provider.config.fetchDetails(liveListings[0]);
+
+        expect(enriched).toBeTruthy();
+        expect(enriched.link).toContain('https://www.deutsche-wohnen.com/');
+        expect(enriched.address).toBeTypeOf('string');
+        expect(enriched.address).not.toBe('');
+        if (enriched.description != null) {
+          expect(enriched.description).toBeTypeOf('string');
+          expect(enriched.description).not.toBe('');
+        }
+      },
+      TEST_TIMEOUT,
+    );
+  });
+});

test/provider/testProvider.json

@@ -56,5 +56,9 @@
   "wohnungsboerse": {
     "url": "https://www.wohnungsboerse.net/searches/index?estate_marketing_types=kauf%2C1&marketing_type=kauf&estate_types%5B0%5D=1&is_rendite=0&estate_id=&zipcodes%5B%5D=&cities%5B%5D=Duesseldorf&districts%5B%5D=&term=D%C3%BCsseldorf&umkreiskm=&pricetext=&minprice=&maxprice=&sizetext=&minsize=&maxsize=&roomstext=&minrooms=&maxrooms=",
     "IsActive": true
+  },
+  "deutscheWohnen": {
+    "url": "https://www.deutsche-wohnen.com/mieten/mietangebote?rentType=miete&city=Berlin&immoType=wohnung",
+    "enabled": true
   }
 }