I’m building a small web scrapper and I have stumbled into the following problem: my applications needs to scrape different parts of a website and put the information into the database. Sometimes it gives crazy results such as duplicated entries, or it returns undefined from a function getPhoto(). However, if I only call that function (and don’t run the rest of the script), it returns a correct result!
I have a for loop, that loops through different URL. It goes to each URL and scrapes the following information: 1. title, 2.description, 3. internal link, 4. calls a function that generates an image according to the title (getPhoto(…) ), 5. saves the results to the DB. Everything happens on the server (I’m using Cron jobs, no client interaction)
for (i = 0; i < AllLinks.length; i++) {
if (AllLinks[i] != undefined && AllLinks[i] != null && sepLink[2] == "www.fly4free.pl") {
var t2 = {
travelTitle: null,
travelTitle2: null,
travelTitle3: null,
travelDescription: null,
travelDescription2: null,
travelDescription3: null,
travelBuy: null,
travelBuy2: null,
travelImage: null
};
var TravelLink1 = AllLinks[i];
result = HTTP.get(AllLinks[i], {});
$ = cheerio.load(result.content);
t2.travelTitle = $('.article__title').text();
t2.travelDescription = $('.article__content').find('p').first().text();
if ($("img[src$='//www.fly4free.pl/wp-content/uploads/2016/09/lotJm.png']").parent().attr('href') != null) {
t2.travelBuy = $("img[src$='//www.fly4free.pl/wp-content/uploads/2016/09/lotJm.png']").parent().attr('href'); // Link to buy
}
if (t2.travelBuy) {
if (t2.travelBuy.split('https://').pop().split('http://').pop() != null) {
t2.travelBuy2 = t2.travelBuy.split('https://').pop().split('http://').pop(); // link ready for DB
} else {
t2.travelBuy2 = t2.travelBuy;
}
}
t2.travelTitle3 = convertCurrencyInText(t2.travelTitle, 'PLN');
t2.travelDescription3 = convertCurrencyInText(t2.travelDescription, 'PLN');
translate(t2.travelTitle3, {from: 'pl', to: 'en'}).then(res => {
t2.travelTitle2 = res.text; // title for DB
if (t2.travelTitle2) { t2.travelImage = getPhoto(t2.travelTitle2); }
translate(t2.travelDescription3, {from: 'pl', to: 'en'}).then(response => {
t2.travelDescription2 = response.text; // description for DB
if (t2.travelDescription2 != null && t2.travelTitle2 != null && t2.travelBuy2 != null && TravelLink1 != null && t2.travelImage != null) {
Links.insert({ title: t2.travelTitle2, description:t2.travelDescription2, image: t2.travelImage, buyLink:t2.travelBuy2, link: TravelLink1, datetime: new Date() });
}
}).catch(err => {
console.error(err);
});
}).catch(err => {
console.error(err);
});
}
}
“AllLinks” contains different URLs. I have problems scraping this URL: http://www.fly4free.pl/na-wakacje-do-toskanii-tanie-loty-do-pizy-z-gdanska-za-170-pln/
getPhoto() function
function getPhoto(title) {
var travelPlace = nlp(title).match('to *').out('text').replace('to','').trim();
if (travelPlace) {var travelPlace2 = travelPlace.split(' '); }
if (travelPlace2) {var travelPlace3 = travelPlace2[0] + "+" + travelPlace2[1]; }
if (travelPlace3) {
var URL = "https://pixabay.com/api/?key="+API_KEY+"&q="+travelPlace3+"&category=travel&orientation=horizontal";
var images = (HTTP.get(URL, {}));
if (images.data.totalHits > 0) {
var imageLink = images.data.hits[0].webformatURL;
return imageLink;
} else if (images.data.totalHits == 0) {
var URL = "https://pixabay.com/api/?key="+API_KEY+"&q="+travelPlace2[0]+"&category=travel&orientation=horizontal";
var images = (HTTP.get(URL, {}));
if (images.data.totalHits > 0) {
var imageLink = images.data.hits[0].webformatURL;
return imageLink;
}
}
} else if (nlp(title).places().data().length > 0) {
var result = nlp(title).places().data()[0].text.replace(/[^a-zA-Z ]/g, "").trim();
var URL = "https://pixabay.com/api/?key="+API_KEY+"&q="+result+"&category=travel&orientation=horizontal";
var images = (HTTP.get(URL, {}));
if (images.data.totalHits > 0) {
var imageLink = images.data.hits[0].webformatURL;
return imageLink;
}
} else {
var title2 = title.replace(/[^a-zA-Z ]/g, "").split(" ");
if (title2) {
for(i = 0; i < title2.length; i++) {
if (cities[title2[i]] == 1) {
var URL = "https://pixabay.com/api/?key="+API_KEY+"&q="+title2[i]+"&category=travel&orientation=horizontal";
var images = (HTTP.get(URL, {}));
if (images.data.totalHits > 0) {
var imageLink = images.data.hits[0].webformatURL;
return imageLink;
}
} else {
var URL = "https://pixabay.com/api/?key="+API_KEY+"&q=travel&category=travel&orientation=horizontal";
var images = (HTTP.get(URL, {}));
if (images.data.totalHits > 0) {
var imageLink = images.data.hits[0].webformatURL;
return imageLink;
}
}
}
}
}
}
I try to console log the results - sometimes I get a correct image from getPhoto(), but an undefined link from t2.travelBuy, sometimes vice versa. Can you tell me what I’m doing wrong? I saw some people are using Promises or async/await functions on that kind of problems. Do you think that would help me? How should I change my code in order to scrape the website without getting “undefined”?
“translate” comes from “google-translate-api” package