Schwarztee/lfs_scraper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implemented rss writer and simple scheduler

Browse Source
This commit is contained in:
Max Hohlfeld 2022-12-23 22:45:48 +01:00
parent 61083535f7
commit cd2dbf7c12
3 changed files with 398 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

317
Cargo.lock generated
View File

@ -2,6 +2,28 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "atom_syndication"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21fb6a0b39c6517edafe46f8137e53c51742425a4dae1c73ee12264a37ad7541"
dependencies = [
"chrono",
"derive_builder",
"diligent-date-parser",
"never",
"quick-xml",
]
[[package]]
name = "autocfg"
version = "1.1.0"
@ -44,6 +66,37 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
dependencies = [
"iana-time-zone",
"num-integer",
"num-traits",
"winapi",
]
[[package]]
name = "clokwerk"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd108d365fcb6d7eddf17a6718eb6a33db18ba4178f8cc6b667f480710f10d76"
dependencies = [
"chrono",
]
[[package]]
name = "codespan-reporting"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
dependencies = [
"termcolor",
"unicode-width",
]
[[package]]
name = "core-foundation"
version = "0.9.3"
@ -60,6 +113,125 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "cxx"
version = "1.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd"
dependencies = [
"cc",
"cxxbridge-flags",
"cxxbridge-macro",
"link-cplusplus",
]
[[package]]
name = "cxx-build"
version = "1.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0"
dependencies = [
"cc",
"codespan-reporting",
"once_cell",
"proc-macro2",
"quote",
"scratch",
"syn",
]
[[package]]
name = "cxxbridge-flags"
version = "1.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59"
[[package]]
name = "cxxbridge-macro"
version = "1.0.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "darling"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]]
name = "derive_builder"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d13202debe11181040ae9063d739fa32cfcaaebe2275fe387703460ae2365b30"
dependencies = [
"derive_builder_macro",
]
[[package]]
name = "derive_builder_core"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66e616858f6187ed828df7c64a6d71720d83767a7f19740b2d1b6fe6327b36e5"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "derive_builder_macro"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73"
dependencies = [
"derive_builder_core",
"syn",
]
[[package]]
name = "diligent-date-parser"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2d0fd95c7c02e2d6c588c6c5628466fff9bdde4b8c6196465e087b08e792720"
dependencies = [
"chrono",
]
[[package]]
name = "encoding_rs"
version = "0.8.31"
@ -261,6 +433,36 @@ dependencies = [
"tokio-native-tls",
]
[[package]]
name = "iana-time-zone"
version = "0.1.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"winapi",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
dependencies = [
"cxx",
"cxx-build",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
version = "0.3.0"
@ -321,14 +523,26 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
name = "lfs_scraper"
version = "0.1.0"
dependencies = [
"clokwerk",
"reqwest",
"rss",
"time",
]
[[package]]
name = "libc"
version = "0.2.138"
version = "0.2.139"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
[[package]]
name = "link-cplusplus"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5"
dependencies = [
"cc",
]
[[package]]
name = "log"
@ -381,6 +595,31 @@ dependencies = [
"tempfile",
]
[[package]]
name = "never"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91"
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.15.0"
@ -475,6 +714,16 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "quick-xml"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8533f14c8382aaad0d592c812ac3b826162128b65662331e1127b45c3d18536b"
dependencies = [
"encoding_rs",
"memchr",
]
[[package]]
name = "quote"
version = "1.0.23"
@ -539,6 +788,18 @@ dependencies = [
"winreg",
]
[[package]]
name = "rss"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acaf1331b7fc4edc3c2920819fee1766c27e8d40da593155832db3d6dea64e92"
dependencies = [
"atom_syndication",
"derive_builder",
"never",
"quick-xml",
]
[[package]]
name = "ryu"
version = "1.0.12"
@ -555,6 +816,12 @@ dependencies = [
"windows-sys 0.36.1",
]
[[package]]
name = "scratch"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2"
[[package]]
name = "security-framework"
version = "2.7.0"
@ -626,6 +893,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "syn"
version = "1.0.107"
@ -651,6 +924,31 @@ dependencies = [
"winapi",
]
[[package]]
name = "termcolor"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
dependencies = [
"winapi-util",
]
[[package]]
name = "time"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
dependencies = [
"serde",
"time-core",
]
[[package]]
name = "time-core"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
[[package]]
name = "tinyvec"
version = "1.6.0"
@ -760,6 +1058,12 @@ dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-width"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
[[package]]
name = "url"
version = "2.3.1"
@ -885,6 +1189,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"

3
Cargo.toml
View File

@ -7,3 +7,6 @@ edition = "2021"
[dependencies]
reqwest = { version = "0.11.13", features = ["blocking"] }
rss = "2"
time = "0.3"
clokwerk = "0.4"

100
src/main.rs
View File

@ -1,41 +1,101 @@
use std::{collections::VecDeque, fs::File};
use time::OffsetDateTime;
use rss::{ChannelBuilder, Item, ItemBuilder};
#[derive(Clone, PartialEq, Eq)]
struct RemainingPlace {
id: String,
description: String,
date: String,
free: usize
free: usize,
}
fn main() -> Result<(), reqwest::Error> {
// const URL: &str = "https://www.lfs.sachsen.de/restplatzboerse-5152.html";
const URL: &str = "http://127.0.0.1:8080/tip.html";
let body = reqwest::blocking::get(URL).unwrap().text().unwrap();
let mut channel = ChannelBuilder::default()
.title(String::from("LFS Restplatzbörse"))
.link(URL.to_string())
.description(String::from("Ein RSS Feed der Restplatzbörse der Landesfeuerwehrschule Sachsen. Nicht offiziell."))
.language(Some("de-DE".to_string()))
.build();
let start = body.find("<tbody").unwrap();
let end = body.find("</tbody>").unwrap();
let mut last_places: Vec<RemainingPlace> = Vec::new();
let mut items: VecDeque<Item> = VecDeque::new();
let table = &body[start..=(end + 7)];
loop {
let body = reqwest::blocking::get(URL).unwrap().text().unwrap();
let mut places: Vec<RemainingPlace> = Vec::new();
let mut iter = table.lines();
let start = body.find("<tbody").unwrap();
let end = body.find("</tbody>").unwrap();
while let Some(line) = iter.next() {
if line.contains("<tr>") {
let id = parse_node(iter.next().unwrap());
let description = parse_node(iter.next().unwrap());
let date = parse_node(iter.next().unwrap());
let free = parse_node(iter.next().unwrap()).parse().unwrap();
let table = &body[start..=(end + 7)];
let place = RemainingPlace { id, description, date, free };
places.push(place);
let mut places: Vec<RemainingPlace> = Vec::new();
let mut iter = table.lines();
while let Some(line) = iter.next() {
if line.contains("<tr>") {
let id = parse_node(iter.next().unwrap());
let description = parse_node(iter.next().unwrap());
let date = parse_node(iter.next().unwrap());
let free = parse_node(iter.next().unwrap()).parse().unwrap();
let place = RemainingPlace {
id,
description,
date,
free,
};
places.push(place);
}
}
}
for place in places {
println!("{} - {} - {} - {}", place.id, place.description, place.date, place.free);
}
let condition = places.len() == last_places.len()
&& places
.iter()
.zip(last_places.iter())
.all(|(one, two)| one == two);
Ok(())
if !condition {
last_places = places.clone();
let title = format!("Restplatzbörse Update - {}", OffsetDateTime::now_utc());
let content = places
.iter()
.map(|place| {
format!(
"{} - {} - {} - {}",
place.id, place.description, place.date, place.free
)
})
.collect::<Vec<String>>()
.join("\n");
let item = ItemBuilder::default()
.title(Some(title))
.content(Some(content))
.build();
items.push_back(item);
if items.len() > 10 {
items.pop_front().unwrap();
}
channel.set_items(items.clone());
let output = File::create("rss.txt").unwrap();
channel.pretty_write_to(output, ' ' as u8, 2).unwrap();
println!("Got Update and wrote to file");
} else {
println!("No update");
}
std::thread::sleep(std::time::Duration::from_secs_f64(60.0));
}
}
fn parse_node(input: &str) -> String {