k3341095/ranking
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

end

Browse Source
This commit is contained in:
kjy
2025-12-16 20:06:49 +09:00
parent 699224fca0
commit b6d69c3623
24 changed files with 425 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/config.ts
View File

@@ -1,4 +1,6 @@
export const realtorIds = ["namyeong00"];
// export const realtorIds = ["namyeong00"];
export const realtorIds = ["diahouse1114"];
// export const realtorIds = ["jdre0125"];
export const tradeTypes = ["A1", "B1", "B2", "B3"];
export const realestateTypes = [
"A01",

15
src/etc/fetch-articles-pc.ts
View File

@@ -1,5 +1,4 @@
import axios from "axios";
import { realtorIds } from "./config";
import axiosRetry from "axios-retry";
axiosRetry(axios, {
@@ -46,6 +45,7 @@ const main = async () => {
const firstResponse = await axios.get(
`https://new.land.naver.com/api/articles?realEstateType=&tradeType=&order=rank&page=1&zoom=0&realtorId=${realtorId}`,
{
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -85,6 +85,7 @@ const main = async () => {
axios.get(
`https://new.land.naver.com/api/articles?realEstateType=&tradeType=&order=rank&page=${page}&zoom=0&realtorId=${realtorId}`,
{
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -98,12 +99,16 @@ const main = async () => {
)
);
const responses = await Promise.all(promises);
const responses = await Promise.allSettled(promises);
responses.forEach((response, index) => {
const articles = response.data.articleList || [];
console.log(`페이지 ${chunk[index]}: ${articles.length}개 매물`);
allArticles.push(...articles);
if (response.status === "fulfilled") {
const articles = response.value.data.articleList || [];
console.log(`페이지 ${chunk[index]}: ${articles.length}개 매물`);
allArticles.push(...articles);
} else {
console.error(`페이지 ${chunk[index]} 요청 실패:`, response.reason);
}
});
// 다음 배치 전에 잠시 대기 (API 부하 방지)

4
src/etc/fetch-detail-disabled.ts
View File

@@ -1,5 +1,5 @@
import { NaverRealEstate } from "./src/services/naver.service";
import prisma from "./src/lib/prisma";
import { NaverRealEstate } from "../services/naver.service";
import { prisma } from "../../lib/prisma";
async function main() {
const realtorId = "a7062525";

22
src/fetch-articles.ts
View File

@@ -8,15 +8,14 @@ async function main() {
realtorId: realtorId,
});
try {
await naver.resetActiveStatus();
let cookie = await naver.getApiCookie();
// 2. DB에서 Seed 불러오기 시도
console.log("Seed 불러오는 중...");
let seed = await naver.fetchSeed(cookie);
console.log("Seed:", seed);
await naver.resetActiveStatus();
// 3. 등록된 매물(Article) 목록 가져오기 (자동으로 DB에 저장됨)
console.log("\n매물 목록 가져오는 중...");
const articles = await naver.getArticlesAndSave();
@@ -33,11 +32,14 @@ async function main() {
console.log(`\n총 ${activeArticles.length}개의 매물 발견`);
for (let i = 0; i < activeArticles.length; i += 30) {
const batch = activeArticles.slice(i, i + 30);
await Promise.all(
batch.map(async (article) => {
await naver.updateBrokerInfo(article.articleNumber, cookie);
// 2. DB에서 Seed 불러오기 시도
console.log("Seed 불러오는 중...");
for (let i = 0; i < activeArticles.length; i += 10) {
const batch = activeArticles.slice(i, i + 10);
await Promise.allSettled(
batch.map((article) => {
return naver.updateBrokerInfo(article.articleNumber, cookie);
})
);
}
@@ -47,5 +49,7 @@ async function main() {
}
}
const sleep = (ms: number) => new Promise((res) => setTimeout(res, ms));
// 실행
main();

3
src/fetch-detailAddress.ts
View File

@@ -1,8 +1,7 @@
import { NaverRealEstate } from "./services/naver.service";
import { realtorIds } from "./config";
async function main() {
const realtorIds = ["namyeong00"];
console.log("detailAddress 없는 매물 목록 가져오는 중...");
for (const realtorId of realtorIds) {

50
src/getRanking.ts Normal file
View File

@@ -0,0 +1,50 @@
import dayjs from "dayjs";
import { prisma } from "../lib/prisma";
import { RankingService } from "./services/ranking.service";
import { NaverRealEstate } from "./services/naver.service";
import { realestateTypes, realtorIds } from "./config";
async function getRanking(articles: RealEstateArticle[], checkDate: Date) {
const { token, cookie } = await this.naver.getRankingToken();
const limit = pLimit(20);
const tasks = articles.map((article) => {
return limit(async () => {
try {
// cortarNo, lgeo 확인
if (!article.cortarNo || !article.lgeo) {
const { cortarNo, lgeo } = await this.updateCortarNoAndLgeo(article);
article.cortarNo = cortarNo;
article.lgeo = lgeo;
}
// 랭킹 조회 및 업데이트
const ranking = await this.naver.getRanking(article, token, cookie);
await prisma.realEstateArticle.update({
where: { id: article.id },
data: {
ranking,
rankCheckDate: checkDate.toISOString(),
},
});
console.log(
`✅ ${article.articleNumber} - 랭킹 업데이트 완료: ${ranking}위`
);
} catch (error) {
console.error(`❌ updateRanking 오류:`, error);
}
});
});
console.log("🔹 예약된 작업 수:", tasks.length);
const results = await Promise.allSettled(tasks);
console.log("🟢 모든 limit 작업 완료:", results.length);
await Bun.sleep(100);
}
(async () => {
const articleNo = process.argv[2];
console.log(articleNo);
process.exit();
const ranking = getRanking();
})();

18
src/lib/prisma.ts
View File

@@ -1,18 +0,0 @@
import { PrismaClient } from "../generated/prisma";
// Prisma Client 싱글톤 인스턴스
const globalForPrisma = globalThis as unknown as {
prisma: PrismaClient | undefined;
};
export const prisma =
globalForPrisma.prisma ??
new PrismaClient({
log: ["error", "warn"],
});
if (process.env.NODE_ENV !== "production") {
globalForPrisma.prisma = prisma;
}
export default prisma;

54
src/app.ts → src/sendTelegram.ts
View File

@@ -1,4 +1,4 @@
import prisma from "./lib/prisma";
import { prisma } from "../lib/prisma";
import dayjs from "dayjs";
import { RankingService } from "./services/ranking.service";
import { NaverRealEstate } from "./services/naver.service";
@@ -21,12 +21,13 @@ const telegramService = new TelegramService(
);
async function main() {
console.time("updateRanking");
await updateRanking();
console.timeEnd("updateRanking");
console.time("sendTelegram");
await sendTelegram();
console.timeEnd("sendTelegram");
try {
console.time("sendTelegram");
await sendTelegram();
console.timeEnd("sendTelegram");
} catch (error) {
console.log(error);
}
}
async function sendTelegram() {
@@ -54,7 +55,9 @@ async function sendTelegram() {
await telegramService.sendDocument(
telegramUser.chatId,
excelFilePath,
`네이버 부동산 매물 목록 (${dayjs().format("YYYY-MM-DD")})`
`${telegramUser.realtorId}_네이버 부동산 매물 목록 (${dayjs().format(
"YYYY-MM-DD"
)})`
);
}
}
@@ -113,9 +116,8 @@ async function createExcelFile(
const cellAddress = `B${index + 2}`; // 헤더가 1행이므로 데이터는 2행부터
const url = `https://fin.land.naver.com/articles/${item.articleNumber}`;
worksheet[cellAddress] = {
t: "s", // string type
v: "열기", // 표시되는 텍스트
l: { Target: url }, // hyperlink
t: "s",
f: `HYPERLINK("https://n8n.abcde.co.kr/webhook/78c0b872-a402-45b4-af20-2b4c88defb58?url=https://fin.land.naver.com/articles/${item.articleNumber}", "열기")`,
};
});
@@ -142,37 +144,13 @@ async function createExcelFile(
XLSX.utils.book_append_sheet(workbook, worksheet, "매물목록");
// 파일 저장
const filePath = `./xlsx/${dayjs().format("YYYY-MM-DD_HH-mm")}_순위_${
telegramUser.site
}_${crypto.randomUUID().slice(0, 8)}.xlsx`;
const filePath = `./xlsx/${telegramUser.realtorId}_${dayjs().format(
"YYYY-MM-DD_HH-mm"
)}_순위_${telegramUser.site}_${crypto.randomUUID().slice(0, 8)}.xlsx`;
XLSX.writeFile(workbook, filePath);
return filePath;
}
async function updateRanking() {
try {
const checkDate = dayjs().toDate();
for (let realtorId of realtorIds) {
const articles = await prisma.realEstateArticle.findMany({
where: {
isActive: true,
realtorId: realtorId,
},
});
const rankingService = new RankingService(
new NaverRealEstate({
realtorId: realtorId,
})
);
await rankingService.updateRanking(articles, checkDate);
}
} catch (error) {
console.log(error);
process.exit();
}
}
main();

47
src/services/naver.service.ts
View File

@@ -5,12 +5,17 @@ import type {
ArticleResponse,
NaverRealEstateConfig,
} from "../types/naver.types";
import prisma from "../lib/prisma";
import { prisma } from "../../lib/prisma";
import type { RealEstateArticle } from "../generated/prisma";
import { findNearest } from "geolib";
import http from "http";
import https from "https";
// axios 인스턴스 생성 및 retry 설정
const axiosInstance = axios.create();
const axiosInstance = axios.create({
// httpAgent: new http.Agent({ keepAlive: false }),
// httpsAgent: new https.Agent({ keepAlive: false }),
});
axiosRetry(axiosInstance, {
retries: 4, // 4번 재시도
retryDelay: (retryCount: number) => {
@@ -27,6 +32,7 @@ export class NaverRealEstate {
async getRankingToken() {
try {
const response = await axios.get("https://new.land.naver.com/houses", {
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -42,6 +48,7 @@ export class NaverRealEstate {
e: "RETAIL",
},
headers: {
connection: "close",
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
"accept-language": "ko;q=0.8",
@@ -159,6 +166,7 @@ export class NaverRealEstate {
method: "post",
maxBodyLength: Infinity,
url: "https://fin.land.naver.com/front-api/v1/realtor/articles",
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -261,6 +269,7 @@ export class NaverRealEstate {
const response = await axiosInstance.post<{
result: ArticleResponse;
}>(`${this.baseUrl}/front-api/v1/realtor/articles`, requestData, {
adapter: "fetch",
headers: {
accept: "application/json, text/plain, */*",
"accept-language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
@@ -309,7 +318,7 @@ export class NaverRealEstate {
}
// API 요청 간격 (선택사항)
const waitTime = 1000 + Math.floor(Math.random() * 1000);
const waitTime = 2000 + Math.floor(Math.random() * 1000);
console.log(`${waitTime}ms 후 다음 페이지 시도...`);
await new Promise((resolve) => setTimeout(resolve, waitTime));
} catch (error) {
@@ -540,6 +549,7 @@ export class NaverRealEstate {
}
}
async resetActiveStatus() {
console.log("초기화하는중");
return prisma.realEstateArticle.updateMany({
where: {
realtorId: this.realtorId,
@@ -577,6 +587,7 @@ export class NaverRealEstate {
const response = await axios.get(
`https://fin.land.naver.com/articles/${articleNumber}`,
{
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -794,6 +805,7 @@ export class NaverRealEstate {
const firstPageResponse = await axiosInstance.get(
`https://m.land.naver.com/agency/info/list?rltrMbrId=${this.realtorId}&tradTpCd=&atclRletTpCd=&tradeTypeChange=false&page=1`,
{
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -803,6 +815,7 @@ export class NaverRealEstate {
},
},
headers: {
connection: "close",
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "ko-KR,ko;q=0.9",
@@ -823,16 +836,26 @@ export class NaverRealEstate {
const totalCnt = firstPageResponse.data.totalCnt;
const totalPage = Math.ceil(totalCnt / 20);
for (let i = 1; i <= totalPage; i += 20) {
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
const BATCH_SIZE = 3;
const DELAY = 10000; // 10초
for (let i = 1; i <= totalPage; i += BATCH_SIZE) {
const group = [];
// i ~ i+19 페이지까지 묶기
for (let page = i; page < i + 20 && page <= totalPage; page++) {
for (let page = i; page < i + BATCH_SIZE && page <= totalPage; page++) {
group.push(updateAddress(page, this.realtorId));
}
// 병렬 실행 후 기다리기
await Promise.all(group);
console.log(`Requesting pages: ${i} ~ ${i + group.length - 1}`);
await Promise.allSettled(group);
console.log(`Batch done. Waiting ${DELAY / 1000}s...`);
await sleep(DELAY);
}
console.log("✅ 모든 페이지 처리 완료");
@@ -844,6 +867,7 @@ export class NaverRealEstate {
const response = await axiosInstance.get(
`https://m.land.naver.com/agency/info/list?rltrMbrId=${realtorId}&tradTpCd=&atclRletTpCd=&tradeTypeChange=false&page=${page}`,
{
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -853,6 +877,7 @@ export class NaverRealEstate {
},
},
headers: {
connection: "close",
accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"accept-language": "ko-KR,ko;q=0.9",
@@ -930,6 +955,7 @@ export class NaverRealEstate {
const url = `https://m.land.naver.com/cluster/clusterList?view=atcl&rletTpCd=APT:OPST:VL:YR:DSD:ABYG:OBYG:JGC:JWJT:DDDGG:SGJT:JGB:OR:SG:SMS:GJCG:GM:TJ:APTHGJ&tradTpCd=A1:B1:B2:B3&z=19&lat=${article.yCoordinate}&lon=${article.xCoordinate}&btm=${btm}&lft=${lft}&top=${top}&rgt=${rgt}&pCortarNo=&addon=COMPLEX&isOnlyIsale=false`;
const response = await axiosInstance.get(url, {
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -939,6 +965,7 @@ export class NaverRealEstate {
},
},
headers: {
Connection: "close",
accept: "application/json, text/javascript, */*; q=0.01",
"accept-language": "ko;q=0.7",
"cache-control": "no-cache",
@@ -1013,6 +1040,7 @@ export class NaverRealEstate {
const url = `https://new.land.naver.com/api/articles?markerId=${article.lgeo}&markerType=LGEOHASH_MIX_ARTICLE&prevScrollTop=0&order=rank&realEstateType=${article.realEstateType}&tradeType=${article.tradeType}&rentPriceMin=0&rentPriceMax=900000000&priceMin=0&priceMax=900000000&areaMin=0&areaMax=900000000&oldBuildYears&recentlyBuildYears&minHouseHoldCount&maxHouseHoldCount&showArticle=false&sameAddressGroup=false&minMaintenanceCost&maxMaintenanceCost&priceType=RETAIL&directions=&page=1&articleState`;
const response = await axiosInstance.get(url, {
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -1022,6 +1050,7 @@ export class NaverRealEstate {
},
},
headers: {
Connection: "close",
accept: "*/*",
"accept-language":
"ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7,la;q=0.6,zh-CN;q=0.5,zh;q=0.4",
@@ -1071,6 +1100,7 @@ export class NaverRealEstate {
const url = `https://fin.land.naver.com/front-api/v1/article/agent?articleNumber=${articleNumber}`;
const response = await axiosInstance.get(url, {
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,
@@ -1140,6 +1170,7 @@ export class NaverRealEstate {
const initialResponse = await axios.get(
"https://fin.land.naver.com/?content=recent",
{
adapter: "fetch",
proxy: {
host: "gw.dataimpulse.com",
port: 823,

39
src/services/ranking.service.ts
View File

@@ -1,6 +1,6 @@
import type { RealEstateArticle } from "../generated/prisma";
import type { NaverRealEstate } from "./naver.service";
import prisma from "../lib/prisma";
import { prisma } from "../../lib/prisma";
import pLimit from "p-limit";
export class RankingService {
@@ -11,9 +11,12 @@ export class RankingService {
async updateRanking(articles: RealEstateArticle[], checkDate: Date) {
const { token, cookie } = await this.naver.getRankingToken();
const limit = pLimit(20);
const tasks = articles.map((article) => {
return limit(async () => {
for (let i = 0; i < articles.length; i += 20) {
const batch = articles.slice(i, i + 20);
console.log(`🚀 ${i + 1} ~ ${i + batch.length}번 매물 처리 시작`);
const tasks = batch.map(async (article) => {
try {
// cortarNo, lgeo 확인
if (!article.cortarNo || !article.lgeo) {
@@ -24,8 +27,10 @@ export class RankingService {
article.lgeo = lgeo;
}
// 랭킹 조회 및 업데이트
// 랭킹 조회
const ranking = await this.naver.getRanking(article, token, cookie);
// DB 업데이트
await prisma.realEstateArticle.update({
where: { id: article.id },
data: {
@@ -34,17 +39,23 @@ export class RankingService {
},
});
console.log(
`✅ ${article.articleNumber} - 랭킹 업데이트 완료: ${ranking}위`
);
} catch (error) {
console.error(`❌ updateRanking 오류:`, error);
console.log(`✅ ${article.articleNumber} - 랭킹 완료 : ${ranking}`);
} catch (err) {
console.error(`❌ 오류: ${article.articleNumber}`, err);
}
});
});
console.log("🔹 예약된 작업 수:", tasks.length);
const results = await Promise.allSettled(tasks);
console.log("🟢 모든 limit 작업 완료:", results.length);
// **이 10개(또는 마지막 batch)가 모두 끝날 때까지 기다림**
await Promise.allSettled(tasks);
// 다음 배치가 남아있다면 딜레이
if (i + 20 < articles.length) {
console.log(`⏸ 10개 처리 완료 → 10초 휴식`);
await Bun.sleep(5000);
}
}
console.log("🎉 전체 랭킹 업데이트 완료!");
await Bun.sleep(100);
}

16
src/services/telegram.service.ts
View File

@@ -76,6 +76,20 @@ export const telegramUsers: TelegramUser[] = [
phone: "010-4199-9650",
realtorId: "namyeong00",
},
// {
// site: "ALL",
// chatId: 6843597951,
// name: "강승원",
// phone: "010-5947-0000",
// realtorId: "diahouse1114",
// },
// {
// site: "ALL",
// chatId: 6843597951,
// name: "강승원",
// phone: "010-5947-0000",
// realtorId: "jdre0125",
// },
];
export const testUsers: TelegramUser[] = [
@@ -192,7 +206,7 @@ export class TelegramService {
})
);
await Promise.all(promises);
await Promise.allSettled(promises);
console.log(`✅ ${users.length}명에게 메시지 브로드캐스트 완료`);
}

34
src/starter.ts
View File

@@ -22,38 +22,28 @@ function runTsFile(filePath: string): Promise<void> {
});
}
// 10시에 실행 - fetch-articles.ts 실행 후 app.ts 실행
schedule.scheduleJob("0 0 10 * * *", async () => {
console.log("Running parserLandList.ts at 10:00 AM");
// 11시에 실행 - fetch-articles -> fetch-detailAddress -> updateRanging -> sendTelegram
schedule.scheduleJob("0 11 * * *", async () => {
console.log("Running scheduled jobs at 11 AM");
try {
await runTsFile("./fetch-articles.ts");
// fetch-articles.ts 완료 후 app.ts 실행
await runTsFile("./fetch-detailAddress.ts");
await runTsFile("./updateRanging.ts");
await runTsFile("./sendTelegram.ts");
} catch (error) {
console.error("Error in scheduled job:", error);
console.error("Error in 11 AM scheduled job:", error);
}
});
// 11시에 실행
schedule.scheduleJob("0 11 * * *", () => {
console.log("Running app.ts at 11 AM");
runTsFile("./app.ts");
});
// 15시에 실행 - fetch-articles.ts 실행 후 app.ts 실행
schedule.scheduleJob("0 0 15 * * *", async () => {
console.log("Running parserLandList.ts at 3:00 PM");
// 16시에 실행 - fetch-articles -> fetch-detailAddress -> updateRanging -> sendTelegram
schedule.scheduleJob("0 16 * * *", async () => {
console.log("Running scheduled jobs at 4 PM");
try {
await runTsFile("./fetch-articles.ts");
// fetch-articles.ts 완료 후 app.ts 실행
await runTsFile("./fetch-detailAddress.ts");
await runTsFile("./updateRanging.ts");
await runTsFile("./sendTelegram.ts");
} catch (error) {
console.error("Error in scheduled job:", error);
console.error("Error in 4 PM scheduled job:", error);
}
});
// 16시에 실행
schedule.scheduleJob("0 16 * * *", () => {
console.log("Running app.ts at 4 PM");
runTsFile("./app.ts");
});

20
src/test.ts Normal file
View File

@@ -0,0 +1,20 @@
import { prisma } from "../lib/prisma";
async function main() {
// Fetch all users with their posts
// const allUsers = await prisma.realEstateArticle.findMany();
console.log("test");
const test = await prisma.realEstateArticle.findMany();
console.log(test);
// console.log("All users:", JSON.stringify(allUsers, null, 2));
}
main()
.then(async () => {
await prisma.$disconnect();
})
.catch(async (e) => {
console.error(e);
await prisma.$disconnect();
process.exit(1);
});

33
src/updateRanking.ts Normal file
View File

@@ -0,0 +1,33 @@
import dayjs from "dayjs";
import { prisma } from "../lib/prisma";
import { RankingService } from "./services/ranking.service";
import { NaverRealEstate } from "./services/naver.service";
import { realestateTypes, realtorIds } from "./config";
async function updateRanking() {
try {
const checkDate = dayjs().toDate();
for (let realtorId of realtorIds) {
const articles = await prisma.realEstateArticle.findMany({
where: {
isActive: true,
realtorId: realtorId,
},
});
const rankingService = new RankingService(
new NaverRealEstate({
realtorId: realtorId,
})
);
await rankingService.updateRanking(articles, checkDate);
}
} catch (error) {
console.log(error);
process.exit();
}
}
(async () => {
await updateRanking();
})();