[컴] node-fetch 사용예제

response text 가 정상적이지 않을때 / cookie 를 set 하는 법

node-fetch 사용예제

fetching 한 data 를 test.txt 로 저장

아래의 url 의 경우는 response.text() 가 제대로 된 결과를 보여주지 못한다. 그래서 Buffer.from(data).toString() 를 이용할 수밖에 없다.

import fetch, { RequestInit } from 'node-fetch';
import fs from 'fs';

async crawlEconomist(): Promise<any> {
    const option: RequestInit = {
        headers: {
            'Host': 'www.economist.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate, br',
            'Referer': 'https://www.economist.com/weeklyedition/2021-03-27',
            'DNT': '1',
            'Connection': 'keep-alive',
            'TE': 'Trailers',
            'Cache-Control': 'no-cache',
        },
    }

    const response = await fetch('https://www.economist.com/leaders/2021/03/27/bright-side-of-the-moonshots', option);
    const data = await response.buffer();

    // array --> Buffer
    const newData = Buffer.from(data);
    fs.writeFileSync("test.txt", newData, {
        encoding: "utf8",
        flag: "a+",
        mode: 0o666
    });

    // array --> Buffer --> string
    const newData2 = Buffer.from(data).toString();
    fs.writeFileSync("test-from-string.txt", newData2, {
        encoding: "utf8",
        flag: "a+",
        mode: 0o666
    });

    console.log('done');

    return {
        data: null,
    };
}

cookie 를 set 하는 법

import fetch, { RequestInit } from 'node-fetch';
import fs from 'fs';

async crawlEconomist(): Promise<any> {
    const option: RequestInit = {
        headers: {
            'Host': 'www.economist.com',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate, br',
            'Referer': 'https://www.economist.com/weeklyedition/2021-03-27',
            'DNT': '1',
            'Connection': 'keep-alive',
            'TE': 'Trailers',
            'Cache-Control': 'no-cache',
        },
    }
    const url = 'https://www.economist.com/leaders/2021/03/27/bright-side-of-the-moonshots';
    const response = await fetch(url, option);

    const cookieRaw = response.headers.raw()['set-cookie'];
    
    // generate cookie string
    const cookie = cookieRaw.map((entry) => {
        const parts = entry.split(';');
        const cookiePart = parts[0];
        return cookiePart;
    }).join(';');
    // set Cookie
    option.headers['Cookie'] = cookie;
    const response = await fetch(url, option);

    console.log('done');

    return {
        data: null,
    };
}

Reference

1. Node.js | fs.writeFileSync() Method - GeeksforGeeks