= env::args().collect();
49 | let timer = time::Instant::now();
50 | let html_dir = &args[1];
51 | let json_dir = &args[2];
52 |
53 | // print all files in the html directory
54 | let _dir = std::fs::read_dir(html_dir).unwrap();
55 | println!("fails and total: {:?}", count_fails(html_dir, json_dir));
56 |
57 | println!("Time elapsed: {:?}", timer.elapsed());
58 | }
59 |
60 | #[cfg(test)]
61 | mod tests {
62 | use super::*;
63 | use pretty_assertions::assert_eq;
64 | const TEST_HTMLS_DIR: &str = "./test_data/pages";
65 | use parser::{
66 | Coordinator, Course, Department, Description, Exam, Faculty, LogisticInformation, Workload,
67 | WorkloadType,
68 | };
69 |
70 | // check that there are files in PAGES_DIR
71 | #[test]
72 | fn test_pages_dir() {
73 | let dir = std::fs::read_dir(TEST_HTMLS_DIR).unwrap();
74 | assert!(dir.count() > 0);
75 | }
76 |
77 | #[test]
78 | fn test_lsls10061_u() {
79 | let html = std::fs::read_to_string(format!("{TEST_HTMLS_DIR}/LSLS10061U.html")).unwrap();
80 | let course = parser::parse_course(&html);
81 | let expected_course = Course {
82 | title: "International Naturforvaltning".to_string(),
83 | info: parser::CourseInformation::new(
84 | "LSLS10061U".to_string(),
85 | 7.5,
86 | vec![parser::Block::Two],
87 | vec![parser::Schedule::B],
88 | vec![parser::Language::Danish],
89 | parser::Duration::One,
90 | vec![parser::Degree::Bachelor],
91 | parser::Capacity(Some(70)),
92 | ),
93 | description: Description {
94 | content: "Skov og naturressourcer globalt:
Globale klima- og vegetationszoner og deres økologi
Verdens naturressourcer, skove, nationalparker og\nnaturområder
Naturbeskyttelse og bevarelsesprogrammer
Globale skovopgørelser, fakta og trends, klassifikationer\u{a0}-\nog problematikker
Udfordringer i forhold til en bæredygtig anvendelse af\nnaturressourcer globalt
\u{a0}
International forvaltning:
International skov og naturressourceforvaltning - politisk,\ninstitutionelt og økonomisk
- Skovpolitik og regeringsførelse
- Internationale konventioner
- Regional EU politik og regelsæt\u{a0}
International skov og naturressourceforvaltning - decentrale\nsystemer\u{a0}
- Agroforestry
- Samfundsbaseret naturressourceforvaltning, indfødte folk og\nviden
Naturressourcebaserede konflikter
Klimaændringer ift. skov og natur
\u{a0}
Produktion, markedet og handel:
International skov og naturressourceforvaltning -\nkommercielt
- Plantage-\u{a0} koncessionsskovbrug
- Dyrkningssystemer og problematikker, optimering og\ngenbevarelse
- International handel med træprodukter \u{a0}
- Ulovlig hugst og handel
- Global og EU markedsbaseret regulering (skovcertificering,\nlegalitetsverificering, FLEGT)
".to_string(),
95 | learning_outcome: "Kursets overordnede formål er at give den enkelte studerende\nviden om de vilkår, der danner rammerne for forvaltningen af skov\nog naturressourcer verden over. Det er også at give en\ngrundlæggende forståelse for hvordan de internationale rammer og\nvilkår har betydning for den måde, som skov og naturressourcer\nforvaltes i Danmark.
Viden:
- Kendskab til vækstvilkår og vegetationstyper globalt
- Kendskab til direkte og underliggende årsager til afskovning\nog over-udnyttelse af ressourcer
- Indsigt i koncessionsskovbrug, plantagedrift og\nagro-forestry
- Kendskab til internationale konventioner og EU lovgivning på\nskov og naturressourceområdet
- Kendsakbs til bevaringsklassifikationer og naturbeskyttelse i\npraksis
- Kendskab til de mest almindelige\u{a0}tømmertræarter
- Kendskab til det internationale markeds betydning for\nforvaltningen af skov- og naturressourcer
- Indsigt i mekanismerne der driver ulovlig hugst og handel med\ntræ
- Indsigt i markedsbaseret regulering
- Kendskab til de mest almindelige certificeringssystemer
- Indsigt i betydningen af\u{a0}klimaforandringer\u{a0}for den\nmåde skov og naturressourcer\u{a0} forvaltes\u{a0}i fremtiden
\u{a0}
Færdigheder:
- Redegøre for hvordan de naturgivne vækstvilkår i forskellige\nvegetationszoner har betydning for den måde som skov – og\nnaturressourcer i disse zoner bør forvaltes
- Redegøre for og diskutere hvordan problemer over-udnyttelse af\nskov og naturressourcer kan afhjælpes\n
\n- Redegøre for hvordan internationale konventioner og EU lovgivning\nhar betydning for forvaltningen af skov og naturressourcer\nlokalt
- Analysere globale markedstrends og forklare hvordan disse har\nbetydning for forvaltningen af skov- og naturressourcer
- Redegøre for de grundlæggende principper bag decentraliseret\nressourceforvaltning
- Identificere de udfordringer, som forvaltningen af\nfællesressourcer kan medføre
- Forklare hvordan markedsbaseret regulering af skov og\nnaturressourceforvaltning fungerer
- Redegøre for de vigtigste drivere bag ulovlig hugst og handel\nsamt hvordan det kan bekæmpes
- Opstille praktiske løsninger for hvordan negative konsekvenser\naf klimaforandringer kan afhjælpes
- Kan argumentere for og imod decentraliseret skov- og\nnaturressourceforvaltning
Kompetencer:
- Arbejde med skov og naturressourceforvaltning under de\nrammevilkår som internationale konventioner og EU lovgivning\ngiver
- Bidrage til at afhjælpe degradering og over-udnyttelse af skov\nog naturressourcer
- Indgå i samarbejder omkring forvaltningen af skov og\nnaturressource i forskellige klimazoner - både nationalt og\ninternationalt
- Tage informerede driftsbeslutninger under hensyntagen til\nnationale og internationale markedstrends
- Arbejde med certificering og legalitetsveriticering
- Integrere klimatilpasningshensyn i forvaltningen af skov og\nnaturressourcer
".to_string(),
96 | recommended_qualifications: Some("Basal forståelse for\nnaturforvaltning og/eller samfundsvidenskab.\n
\nSprogkundskaber til at kunne læse og forstå engelsk faglitteratur.\n
\nTilnærmelsesvis alt litteratur er på engelsk.".to_string()),
97 | summary: "Skov og naturressourcer globalt:Globale klima- og vegetationszoner og deres økologiVerdens naturressourcer, skove, nationalparker og\nnaturområderNaturbeskyttelse og bevarelsesprogrammerGlobale skovopgørelser, fakta og trends, klassifikationer\u{a0}-\nog problematikkerUdfordringer i forhold til en bæredygt".to_string(),
98 | },
99 | exams: vec![Exam::Oral(Some(30))],
100 | /*
101 | * > workloads: [
102 | > Workload {
103 | > workload_type: Lectures,
104 | > hours: 98.0,
105 | > },
106 | > Workload {
107 | > workload_type: Preparation,
108 | > hours: 97.0,
109 | > },
110 | > Workload {
111 | > workload_type: ProjectWork,
112 | > hours: 8.0,
113 | > },
114 | > Workload {
115 | > workload_type: Exam,
116 | > hours: 1.0,
117 | > },
118 | > ],
119 | */
120 | workloads: vec![
121 | Workload::new(WorkloadType::Lectures, 98.0),
122 | Workload::new(WorkloadType::Preparation, 97.0),
123 | Workload::new(WorkloadType::ProjectWork, 8.0),
124 | Workload::new(WorkloadType::Exam, 1.0),
125 | ],
126 |
127 | logistics: LogisticInformation::new(
128 | vec![Department::GeosciencesAndNaturalResourceManagement],
129 | Faculty::Science,
130 | vec![Coordinator::new(
131 | "Kirsten Carlsen".into(),
132 | "bdk748@alumni.ku.dk".into(),
133 | )],
134 | ),
135 | };
136 | assert_eq!(expected_course, course.unwrap());
137 | }
138 |
139 | // We need to ignore the duration if the course is known to be a summer course.
140 | #[ignore]
141 | #[test]
142 | fn test_nbik15000_u() {
143 | let html = std::fs::read_to_string(format!("{TEST_HTMLS_DIR}/NBIK15000U.html")).unwrap();
144 | let course = parser::parse_course(&html);
145 | let expected_course = Course {
146 | title: "BAdvanced Plant Identification".to_string(),
147 | info: parser::CourseInformation::new(
148 | "NBIK15000U".to_string(),
149 | 7.5,
150 | vec![parser::Block::Summer],
151 | vec![parser::Schedule::B], // doesnt exist
152 | vec![parser::Language::English],
153 | parser::Duration::One,
154 | vec![parser::Degree::Master],
155 | parser::Capacity(Some(16)),
156 | ),
157 | description: Description {
158 | content: "".to_string(),
159 | learning_outcome: "".to_string(),
160 | recommended_qualifications: Some("".to_string()),
161 | summary: "".to_string(),
162 | },
163 | exams: vec![Exam::Oral(Some(30))],
164 | workloads: Vec::new(),
165 | logistics: LogisticInformation::new(Vec::new(), Faculty::Science, Vec::new()),
166 | };
167 | assert_eq!(expected_course, course.unwrap());
168 | }
169 | }
170 |
--------------------------------------------------------------------------------
/backend/rust_parser/src/parser.rs:
--------------------------------------------------------------------------------
1 | use anyhow::{bail, ensure, Context, Result};
2 | use serde::Serialize;
3 | use tl::VDom;
4 |
5 | use crate::parser::course_information::parse_course_info;
6 | pub mod course_information;
7 |
8 | use crate::parser::exam_information::parse_course_exams;
9 | pub mod exam_information;
10 |
11 | use crate::parser::logistic_information::parse_logistic_info;
12 | pub mod logistic_information;
13 |
14 | use crate::parser::workload_information::parse_workloads;
15 | pub mod workload_information;
16 |
17 | use crate::parser::content_serialiser::grab_htmls;
18 | pub mod content_serialiser;
19 |
20 | #[derive(Debug, PartialEq, Serialize)]
21 | pub struct Course {
22 | pub title: String,
23 | pub info: CourseInformation,
24 | pub logistics: LogisticInformation,
25 | pub workloads: Vec,
26 | pub exams: Vec,
27 | pub description: Description,
28 | }
29 |
30 | #[derive(Debug)]
31 | pub enum CourseLanguage {
32 | English,
33 | Danish,
34 | }
35 |
36 | #[derive(Debug, PartialEq, Serialize)]
37 | pub struct CourseInformation {
38 | pub id: String,
39 | ects: f32,
40 | block: Vec,
41 | schedule: Vec,
42 | language: Vec,
43 | duration: Duration,
44 | degree: Vec,
45 | capacity: Capacity,
46 | }
47 |
48 | #[derive(Debug, PartialEq, Serialize)]
49 | pub enum Department {
50 | // Faculty of Science
51 | PlantAndEnvironmentalScience,
52 | Biology,
53 | ComputerScience,
54 | FoodAndResourceEconomics,
55 | FoodScience,
56 | GeosciencesAndNaturalResourceManagement,
57 | NutritionExerciseAndSports,
58 | Mathematics,
59 | ScienceEducation,
60 | // PlantAndEnvironmentalSciences, this never occurs as of end of 2023
61 | Chemistry,
62 | NielsBohrInstitute,
63 | NaturalHistoryMuseumOfDenmark,
64 | VeterinaryAndAnimalSciences,
65 | BiomedicalSciences,
66 | PublicHealth,
67 | DrugDesignAndPharmacology,
68 | CellularAndMolecularMedicine,
69 | Pharmacy,
70 | Globe,
71 | }
72 | impl Department {
73 | fn from_str(s: &str) -> Result {
74 | match s.replace('\n', " ").as_str() {
75 | "Department of Computer Science" | "Datalogisk Institut" => {
76 | Ok(Department::ComputerScience)
77 | }
78 | "Institut for Idræt og Ernæring" | "Department of Nutrition, Exercise and Sports" => {
79 | Ok(Department::NutritionExerciseAndSports)
80 | }
81 | "Statens Naturhistoriske Museum" | "The Natural History Museum of Denmark" => {
82 | Ok(Department::NaturalHistoryMuseumOfDenmark)
83 | }
84 | "Institut for Plante- og Miljøvidenskab"
85 | | "Department of Plant and Environmental Sciences" => {
86 | Ok(Department::PlantAndEnvironmentalScience)
87 | }
88 | "Institut for Matematiske Fag" | "Department of Mathematical Sciences" => {
89 | Ok(Department::Mathematics)
90 | }
91 | "Niels Bohr Institutet" | "The Niels Bohr Institute" => {
92 | Ok(Department::NielsBohrInstitute)
93 | }
94 | "Institut for Geovidenskab og Naturforvaltning"
95 | | "Department of Geoscience and Natural Resource Management" => {
96 | Ok(Department::GeosciencesAndNaturalResourceManagement)
97 | }
98 | "Institut for Naturfagenes Didaktik" | "Department of Science Education" => {
99 | Ok(Department::ScienceEducation)
100 | }
101 | "Institut for Fødevare- og Ressourceøkonomi"
102 | | "Department of Food and Resource Economics" => {
103 | Ok(Department::FoodAndResourceEconomics)
104 | }
105 | "Institut for Fødevarevidenskab" | "Department of Food Science" => {
106 | Ok(Department::FoodScience)
107 | }
108 | "Kemisk Institut" | "Department of Chemistry" => Ok(Department::Chemistry),
109 | "Biologisk Institut" | "Department of Biology" => Ok(Department::Biology),
110 | "Department of Veterinary and Animal Sciences"
111 | | "Institut for Veterinær- og Husdyrvidenskab (IVH)" => {
112 | Ok(Department::VeterinaryAndAnimalSciences)
113 | }
114 | "Department of Biomedical Sciences" => Ok(Department::BiomedicalSciences),
115 | "Department of Pharmacy" => Ok(Department::Pharmacy),
116 | "Institut for Lægemiddeldesign og Farmakologi"
117 | | "Department of Drug Design and Pharmacology" => {
118 | Ok(Department::DrugDesignAndPharmacology)
119 | }
120 | "Department of Cellular and Molecular Medicine" => {
121 | Ok(Department::CellularAndMolecularMedicine)
122 | }
123 | "Department of Public Health" => Ok(Department::PublicHealth),
124 | "Institut for Nordiske Studier og Sprogvidenskab" => {
125 | bail!("Nordic studies not supported ")
126 | }
127 | "GLOBE Institute" => Ok(Department::Globe),
128 | _ => bail!("Unknown department: {}", s),
129 | }
130 | }
131 | }
132 |
133 | #[derive(Debug, PartialEq, Serialize)]
134 | pub enum Faculty {
135 | Science,
136 | }
137 |
138 | #[derive(Debug, PartialEq, Serialize)]
139 | pub struct Coordinator {
140 | name: String,
141 | email: String,
142 | }
143 | impl Coordinator {
144 | pub fn new(name: String, email: String) -> Self {
145 | Self { name, email }
146 | }
147 | }
148 |
149 | #[derive(Debug, PartialEq, Serialize)]
150 | pub struct LogisticInformation {
151 | departments: Vec,
152 | faculty: Faculty,
153 | coordinators: Vec,
154 | }
155 | impl LogisticInformation {
156 | pub fn new(
157 | departments: Vec,
158 | faculty: Faculty,
159 | coordinators: Vec,
160 | ) -> Self {
161 | Self {
162 | departments,
163 | faculty,
164 | coordinators,
165 | }
166 | }
167 | }
168 |
169 | impl CourseInformation {
170 | #[allow(clippy::too_many_arguments)]
171 | pub fn new(
172 | id: String,
173 | ects: f32,
174 | block: Vec,
175 | schedule: Vec,
176 | language: Vec,
177 | duration: Duration,
178 | degree: Vec,
179 | capacity: Capacity,
180 | ) -> Self {
181 | Self {
182 | id,
183 | ects,
184 | block,
185 | schedule,
186 | language,
187 | duration,
188 | degree,
189 | capacity,
190 | }
191 | }
192 | }
193 |
194 | #[derive(Debug, PartialEq, Serialize)]
195 | pub enum Block {
196 | One,
197 | Two,
198 | Three,
199 | Four,
200 | Summer,
201 | Other(String),
202 | }
203 |
204 | #[derive(Debug, PartialEq, Serialize)]
205 | pub enum Schedule {
206 | A,
207 | B,
208 | C,
209 | D,
210 | OutsideOfSchedule,
211 | Other(String),
212 | }
213 |
214 | #[derive(Debug, PartialEq, Serialize)]
215 | pub enum Language {
216 | Danish,
217 | English,
218 | }
219 |
220 | #[derive(Debug, Eq, PartialEq, Serialize)]
221 | pub enum Duration {
222 | One,
223 | Two,
224 | DependsOnEcts,
225 | Custom(String),
226 | }
227 |
228 | #[derive(Debug, PartialEq, Eq, Ord, PartialOrd, Serialize)]
229 | pub enum Degree {
230 | Phd,
231 | Bachelor,
232 | Master,
233 | Propædeutik,
234 | }
235 |
236 | #[derive(Debug, PartialEq, Serialize)]
237 | pub struct Capacity(pub Option);
238 |
239 | #[derive(Debug, PartialEq, Serialize)]
240 | pub enum Exam {
241 | Oral(Option),
242 | Written(Option),
243 | Assignment(Option),
244 | ContinuousAssessment,
245 | Other,
246 | }
247 |
248 | #[derive(Debug, PartialEq, Serialize)]
249 | pub enum WorkloadType {
250 | Exam,
251 | ELearning,
252 | Laboratory,
253 | StudyGroup,
254 | TheoryExercises,
255 | FieldWork,
256 | Preparation,
257 | ExamPreparation,
258 | Excursions,
259 | Lectures,
260 | PracticalExercises,
261 | ProjectWork,
262 | Exercises,
263 | Guidance,
264 | ClassInstruction,
265 | PracticalTraining,
266 | Seminar,
267 | }
268 | impl WorkloadType {
269 | fn from_str(s: &str) -> Result {
270 | match s {
271 | "Forelæsninger" | "Lectures" => Ok(WorkloadType::Lectures),
272 | "Forberedelse (anslået)" | "Preparation" => Ok(WorkloadType::Preparation),
273 | "Theory exercises" | "Teoretiske øvelser" => Ok(WorkloadType::TheoryExercises),
274 | "Project work" | "Projektarbejde" => Ok(WorkloadType::ProjectWork),
275 | "Øvelser" | "Exercises" => Ok(WorkloadType::Exercises),
276 | "Eksamen" | "Exam" => Ok(WorkloadType::Exam),
277 | "Eksamensforberedelse" | "Exam Preparation" => Ok(WorkloadType::ExamPreparation),
278 | "Holdundervisning" | "Class Instruction" => Ok(WorkloadType::ClassInstruction),
279 | "Practical exercises" | "Praktiske øvelser" => Ok(WorkloadType::PracticalExercises),
280 | "E-Learning" | "E-læring" => Ok(WorkloadType::ELearning),
281 | "Feltarbejde" | "Field Work" => Ok(WorkloadType::FieldWork),
282 | "Studiegrupper" | "Study Groups" => Ok(WorkloadType::StudyGroup),
283 | "Seminar" => Ok(WorkloadType::Seminar),
284 | "Ekskursioner" | "Excursions" => Ok(WorkloadType::Excursions),
285 | "Laboratorie" | "Laboratory" => Ok(WorkloadType::Laboratory),
286 | "Praktik" | "Practical Training" => Ok(WorkloadType::PracticalTraining),
287 |
288 | "Guidance" | "Vejledning" => Ok(WorkloadType::Guidance),
289 | _ => bail!("Unknown workload type: {}", s),
290 | }
291 | }
292 | }
293 |
294 | #[derive(Debug, PartialEq, Serialize)]
295 | pub struct Workload {
296 | workload_type: WorkloadType,
297 | hours: f32,
298 | }
299 | impl Workload {
300 | pub fn new(workload_type: WorkloadType, hours: f32) -> Self {
301 | Self {
302 | workload_type,
303 | hours,
304 | }
305 | }
306 | }
307 |
308 | #[derive(Debug, PartialEq, Serialize)]
309 | pub struct Description {
310 | pub content: String,
311 | pub learning_outcome: String,
312 | pub recommended_qualifications: Option,
313 | pub summary: String,
314 | }
315 |
316 | pub fn parse_course(html: &str) -> Result {
317 | let dom = tl::parse(html, tl::ParserOptions::default())?;
318 | let content = dom.get_element_by_id("content");
319 | let title = parse_title(&dom)?;
320 |
321 | ensure!(
322 | content.is_some(),
323 | "Unable to find content element, this should not happen"
324 | );
325 | let info = parse_course_info(&dom).context(format!("Unable to parse course: {title}"))?;
326 | let logistic_info = parse_logistic_info(&dom).context(format!(
327 | "Unable to parse logistic information for course: {title}"
328 | ))?;
329 |
330 | let exam_info = parse_course_exams(&dom).context(format!(
331 | "Unable to parse exam information for course: {title}"
332 | ))?;
333 |
334 | let workload_info = parse_workloads(&dom).context(format!(
335 | "Unable to parse workload information for course: {title}"
336 | ))?;
337 |
338 | let html_info = grab_htmls(&dom).context(format!(
339 | "Unable to grab html information for course: {title}"
340 | ))?;
341 |
342 | Ok(Course {
343 | title,
344 | info,
345 | logistics: logistic_info,
346 | exams: exam_info,
347 | workloads: workload_info,
348 | description: html_info,
349 | })
350 | }
351 | fn parse_title(dom: &VDom) -> Result {
352 | let title = dom
353 | .get_elements_by_class_name("courseTitle")
354 | .next()
355 | .context("Unable to find course title")
356 | .and_then(|elem| {
357 | elem.get(dom.parser())
358 | .context("Unable to grab parser for the dom, this should not happen")
359 | .map(|tag| tag.inner_text(dom.parser()))
360 | });
361 |
362 | let binding = title
363 | .unwrap_or_else(|_| "Error unwrapping html title".into())
364 | .replace(['\u{a0}', '\n'], " ");
365 |
366 | // Then split them
367 | let res: Vec<&str> = binding.split_whitespace().collect();
368 |
369 | // Return only the part of the title without the course code
370 | ensure!(
371 | res.len() >= 2,
372 | "Title does not conform to expected structure: "
373 | );
374 |
375 | Ok(res[1..].join(" "))
376 | }
377 |
--------------------------------------------------------------------------------