├── BE
├── .gitignore
├── .env
├── requirements.txt
├── app.py
└── chatbot
│ └── routes.py
├── FE
├── src
│ ├── constants
│ │ ├── index.ts
│ │ └── constants.ts
│ ├── vite-env.d.ts
│ ├── pages
│ │ ├── Landing.tsx
│ │ ├── Dashboard.tsx
│ │ └── Pages.tsx
│ ├── main.tsx
│ ├── index.css
│ ├── App.tsx
│ ├── routes.ts
│ ├── layouts
│ │ ├── Chatbot
│ │ │ ├── server.ts
│ │ │ ├── components
│ │ │ │ ├── Message.tsx
│ │ │ │ └── Input.tsx
│ │ │ ├── styles.ts
│ │ │ └── Chatbot.tsx
│ │ ├── Sidebar
│ │ │ ├── styles.ts
│ │ │ ├── components
│ │ │ │ └── History.tsx
│ │ │ └── Sidebar.tsx
│ │ ├── LandingBody
│ │ │ ├── styles.ts
│ │ │ └── LandingBody.tsx
│ │ ├── Footer
│ │ │ └── Footer.tsx
│ │ └── Header
│ │ │ ├── styles.ts
│ │ │ └── Header.tsx
│ ├── contexts
│ │ ├── usePersonalization.tsx
│ │ └── useIsDarkTheme.tsx
│ ├── theme
│ │ └── index.ts
│ └── data
│ │ └── data.ts
├── .env
├── public
│ ├── Jenkins_logo.ico
│ ├── 3-dots-fade.svg
│ └── Jenkins_logo.svg
├── vite.config.ts
├── tsconfig.node.json
├── .gitignore
├── index.html
├── .eslintrc.cjs
├── tsconfig.json
├── package.json
└── README.md
├── .gitattributes
├── images
├── llama.gif
├── llama2.png
└── llama2.webp
├── JenAi Final Document.docx
├── JenAi Final Document.pdf
├── datasets
├── raw
│ ├── Jenkins Docs.xlsx
│ └── QueryResults.csv
├── Jenkins Docs QA.csv
├── QueryResultsUpdated.csv
└── Community Questions Refined.csv
├── src
├── data preprocessing
│ ├── utils.py
│ ├── preprocessing.py
│ └── preprocessing.ipynb
└── data collection
│ ├── utils.py
│ ├── qa-article-to-qa-csv-pairs.ipynb
│ ├── parse-jenkins-community.ipynb
│ └── refine-html-tags.ipynb
├── LICENSE
├── .gitignore
└── README.md
/BE/.gitignore:
--------------------------------------------------------------------------------
1 | Llama-2-7b-chat-finetune/
2 |
--------------------------------------------------------------------------------
/FE/src/constants/index.ts:
--------------------------------------------------------------------------------
1 | export * from "./constants";
2 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.csv filter=lfs diff=lfs merge=lfs -text
2 |
--------------------------------------------------------------------------------
/FE/src/vite-env.d.ts:
--------------------------------------------------------------------------------
1 | ///
11 | for tag in soup.find_all(True):
12 | if tag.name != 'code':
13 | tag.unwrap() # Remove the tag, keep its contents
14 |
15 | return str(soup)
16 |
17 |
--------------------------------------------------------------------------------
/FE/public/3-dots-fade.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/FE/src/index.css:
--------------------------------------------------------------------------------
1 | body {
2 | margin: 0;
3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
5 | sans-serif;
6 | -webkit-font-smoothing: antialiased;
7 | -moz-osx-font-smoothing: grayscale;
8 | height: 100vh;
9 | }
10 |
11 | body > div {
12 | height: 100%;
13 | }
14 |
15 | code {
16 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
17 | monospace;
18 | }
19 |
20 |
--------------------------------------------------------------------------------
/FE/.eslintrc.cjs:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | root: true,
3 | env: { browser: true, es2020: true },
4 | extends: [
5 | 'eslint:recommended',
6 | 'plugin:@typescript-eslint/recommended',
7 | 'plugin:react-hooks/recommended',
8 | ],
9 | ignorePatterns: ['dist', '.eslintrc.cjs'],
10 | parser: '@typescript-eslint/parser',
11 | plugins: ['react-refresh'],
12 | rules: {
13 | 'react-refresh/only-export-components': [
14 | 'warn',
15 | { allowConstantExport: true },
16 | ],
17 | },
18 | }
19 |
--------------------------------------------------------------------------------
/FE/src/App.tsx:
--------------------------------------------------------------------------------
1 | import React from "react";
2 | import Pages from "./pages/Pages";
3 | import { IsDarkThemeProvider } from "./contexts/useIsDarkTheme";
4 | import { PersonalizationProvider } from "./contexts/usePersonalization";
5 |
6 | function App() {
7 | return (
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 | );
16 | }
17 |
18 | export default App;
19 |
--------------------------------------------------------------------------------
/FE/src/routes.ts:
--------------------------------------------------------------------------------
1 | import Dashboard from "./pages/Dashboard";
2 | import Landing from "./pages/Landing";
3 |
4 | export type RouteSchema = {
5 | name: string;
6 | key: string;
7 | route: string;
8 | component?: React.FunctionComponent;
9 | };
10 |
11 | export const routes: RouteSchema[] = [
12 | {
13 | name: "Dashboard",
14 | key: "dashboard",
15 | route: "/dashboard/:chatId?",
16 | component: Dashboard,
17 | },
18 | {
19 | name: "Landging",
20 | key: "landing",
21 | route: "/landing",
22 | component: Landing,
23 | },
24 | ];
25 |
--------------------------------------------------------------------------------
/FE/src/layouts/Chatbot/server.ts:
--------------------------------------------------------------------------------
1 | import axios from "axios";
2 |
3 | interface ApiResponse {
4 | prediction: string;
5 | }
6 |
7 | const chatEndpoint =
8 | import.meta.env.VITE_SERVER_URL + import.meta.env.VITE_CHAT_ENDPOINT;
9 |
10 | export const sendQuery = (query: string, personalization:string): Promise => {
11 | return axios
12 | .post(chatEndpoint, { text: query, persona: personalization })
13 | .then((response) => response.data.prediction)
14 | .catch((error) => {
15 | console.error("Error sending query:", error);
16 | throw error;
17 | });
18 | };
19 |
--------------------------------------------------------------------------------
/BE/app.py:
--------------------------------------------------------------------------------
1 | from flask import Flask
2 | from chatbot.routes import chatbot_bp
3 | from flask_cors import CORS
4 | import os
5 | from dotenv import load_dotenv
6 |
7 | # Load environment variables from a .env file
8 | load_dotenv('.env')
9 |
10 | app = Flask(__name__)
11 |
12 | # Load configuration
13 | env = os.environ.get('FLASK_ENV', 'development')
14 |
15 | # Configure CORS
16 | CORS(app)
17 |
18 | # Register the blueprint with configurable URL prefix
19 | app.register_blueprint(chatbot_bp, url_prefix=os.getenv('CHATBOT_URL_PREFIX'))
20 |
21 | if __name__ == '__main__':
22 | app.run(host=os.getenv('FLASK_RUN_HOST'), port=int(os.getenv('FLASK_RUN_PORT')))
23 |
--------------------------------------------------------------------------------
/FE/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "ES2020",
4 | "useDefineForClassFields": true,
5 | "lib": ["ES2020", "DOM", "DOM.Iterable"],
6 | "module": "ESNext",
7 | "skipLibCheck": true,
8 |
9 | /* Bundler mode */
10 | "moduleResolution": "bundler",
11 | "allowImportingTsExtensions": true,
12 | "resolveJsonModule": true,
13 | "isolatedModules": true,
14 | "noEmit": true,
15 | "jsx": "react-jsx",
16 |
17 | /* Linting */
18 | "strict": true,
19 | "noUnusedLocals": true,
20 | "noUnusedParameters": true,
21 | "noFallthroughCasesInSwitch": true
22 | },
23 | "include": ["src"],
24 | "references": [{ "path": "./tsconfig.node.json" }]
25 | }
26 |
--------------------------------------------------------------------------------
/FE/src/layouts/Sidebar/styles.ts:
--------------------------------------------------------------------------------
1 | import { ListItem } from "@mui/material";
2 | import { styled } from "@mui/material/styles";
3 |
4 | interface ListItemProps {
5 | disablePadding?: boolean;
6 | contained?: string;
7 | }
8 |
9 | export const CustomListItem = styled(ListItem)(
10 | ({ theme, contained }) => ({
11 | backgroundColor:
12 | contained === "true" ? theme.palette.grey[900] : "transparent",
13 | })
14 | );
15 |
16 | export const DrawerHeader = styled("div")(({ theme }) => ({
17 | display: "flex",
18 | alignItems: "center",
19 | padding: theme.spacing(0, 1),
20 | // necessary for content to be below app bar
21 | ...theme.mixins.toolbar,
22 | justifyContent: "space-between",
23 | }));
24 |
--------------------------------------------------------------------------------
/FE/src/pages/Dashboard.tsx:
--------------------------------------------------------------------------------
1 | import Sidebar from "../layouts/Sidebar/Sidebar";
2 | import Chatbot from "../layouts/Chatbot/Chatbot";
3 |
4 | import { Box } from "@mui/material";
5 | import { useState } from "react";
6 | import Header from "../layouts/Header/Header";
7 |
8 | const Dashboard = () => {
9 | const [open, setOpen] = useState(true);
10 |
11 | const handleDrawerOpen = () => {
12 | setOpen(true);
13 | };
14 |
15 | const handleDrawerClose = () => {
16 | setOpen(false);
17 | };
18 |
19 | return (
20 |
21 |
22 |
23 |
24 |
25 | );
26 | };
27 |
28 | export default Dashboard;
29 |
--------------------------------------------------------------------------------
/FE/src/layouts/LandingBody/styles.ts:
--------------------------------------------------------------------------------
1 | import { Box, Typography as MUITypography } from "@mui/material";
2 | import { styled } from "@mui/material/styles";
3 |
4 | type InputProps = {
5 | color?: string;
6 | };
7 |
8 | export const ButtonsWrapper = styled(Box)(() => ({
9 | display: "flex",
10 | alignItems: "center",
11 | justifyContent: "center",
12 | gap: "1rem",
13 | }));
14 |
15 | export const Container = styled(Box)(({ theme }) => ({
16 | height: "100vh",
17 | display: "flex",
18 | flexDirection: "column",
19 | alignItems: "center",
20 | justifyContent: "center",
21 | gap: "2rem",
22 | width: "60%",
23 | margin: "auto",
24 | textAlign: "center",
25 | color: theme.palette.text.primary,
26 | }));
27 |
28 | export const Typography = styled(MUITypography)(({ color }) => ({
29 | color,
30 | display: "inline",
31 | }));
32 |
--------------------------------------------------------------------------------
/FE/src/contexts/usePersonalization.tsx:
--------------------------------------------------------------------------------
1 | import { createContext, useContext, useState } from "react";
2 |
3 | type ProviderProps = {
4 | children: React.ReactNode;
5 | };
6 |
7 | type ContextValue = {
8 | text: string;
9 | handleUpdate: (text: string) => void;
10 | };
11 |
12 | const PersonalizationContext = createContext({
13 | text: "",
14 | handleUpdate: () => {},
15 | });
16 |
17 | export function PersonalizationProvider({ children }: ProviderProps) {
18 | const [text, setText] = useState("");
19 |
20 | const handleUpdate = (text: string) => {
21 | setText(text);
22 | };
23 |
24 | return (
25 |
26 | {children}
27 |
28 | );
29 | }
30 |
31 | export function usePersonalization() {
32 | return useContext(PersonalizationContext);
33 | }
34 |
--------------------------------------------------------------------------------
/FE/src/contexts/useIsDarkTheme.tsx:
--------------------------------------------------------------------------------
1 | import { createContext, useContext, useState } from "react";
2 |
3 | type ProviderProps = {
4 | children: React.ReactNode;
5 | };
6 |
7 | type ContextValue = {
8 | isDarkTheme: boolean;
9 | handleThemeChange: () => void;
10 | };
11 |
12 | const IsDarkThemeContext = createContext({
13 | isDarkTheme: false,
14 | handleThemeChange: () => {},
15 | });
16 |
17 | export function IsDarkThemeProvider({ children }: ProviderProps) {
18 | const [isDarkTheme, setIsDarkTheme] = useState(true);
19 |
20 | const handleThemeChange = () => {
21 | setIsDarkTheme(!isDarkTheme);
22 | };
23 |
24 | return (
25 |
26 | {children}
27 |
28 | );
29 | }
30 |
31 | export function useIsDarkTheme() {
32 | return useContext(IsDarkThemeContext);
33 | }
34 |
--------------------------------------------------------------------------------
/FE/src/layouts/Footer/Footer.tsx:
--------------------------------------------------------------------------------
1 | import { Box, Container, Grid, Typography, useTheme } from "@mui/material";
2 |
3 | const Footer = () => {
4 | const theme = useTheme();
5 |
6 | return (
7 |
8 |
9 |
10 |
11 |
16 | Jenkins Chatbot
17 |
18 |
19 |
20 |
21 | {`GSoC'24 | Jenkins | Enhancing LLM Project`}
22 |
23 |
24 |
25 |
26 |
27 | );
28 | };
29 |
30 | export default Footer;
31 |
--------------------------------------------------------------------------------
/FE/src/layouts/Header/styles.ts:
--------------------------------------------------------------------------------
1 | import { styled } from "@mui/material/styles";
2 | import MuiAppBar, { AppBarProps as MuiAppBarProps } from "@mui/material/AppBar";
3 |
4 | const drawerWidth = 240;
5 |
6 | interface AppBarProps extends MuiAppBarProps {
7 | open?: boolean;
8 | }
9 |
10 | export const AppBar = styled(MuiAppBar, {
11 | shouldForwardProp: (prop) => prop !== "open",
12 | })(({ theme, open }) => ({
13 | backgroundColor: theme.palette.secondary.main,
14 | paddingRight: 30,
15 | paddingLeft: 30,
16 | height: 80,
17 | justifyContent: "center",
18 | transition: theme.transitions.create(["margin", "width"], {
19 | easing: theme.transitions.easing.sharp,
20 | duration: theme.transitions.duration.leavingScreen,
21 | }),
22 | ...(open && {
23 | width: `calc(100% - ${drawerWidth}px)`,
24 | marginLeft: `${drawerWidth}px`,
25 | transition: theme.transitions.create(["margin", "width"], {
26 | easing: theme.transitions.easing.easeOut,
27 | duration: theme.transitions.duration.enteringScreen,
28 | }),
29 | }),
30 | }));
31 |
--------------------------------------------------------------------------------
/FE/src/theme/index.ts:
--------------------------------------------------------------------------------
1 | import { createTheme, responsiveFontSizes } from "@mui/material/styles";
2 |
3 | // Define light theme
4 | const lightTheme = responsiveFontSizes(
5 | createTheme({
6 | typography: {},
7 | palette: {
8 | mode: "light",
9 | primary: {
10 | main: "#bdbdbd",
11 | light: "#ecedf2",
12 | dark: "#676767",
13 | },
14 | secondary: {
15 | main: "#003b6d",
16 | light: "#669acc",
17 | },
18 | text: {
19 | primary: "#000000",
20 | secondary: "#ffffff",
21 | },
22 | },
23 | })
24 | );
25 |
26 | // Define dark theme
27 | const darkTheme = responsiveFontSizes(
28 | createTheme({
29 | typography: {},
30 | palette: {
31 | mode: "dark",
32 | primary: {
33 | main: "#f50057",
34 | dark: "#f50057",
35 | },
36 | secondary: {
37 | main: "#000000",
38 | },
39 | text: {
40 | primary: "#ffffff",
41 | secondary: "#000000",
42 | },
43 | },
44 | })
45 | );
46 |
47 | export { lightTheme, darkTheme };
48 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2024 Nour Ziad Almulhem
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/FE/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "fe",
3 | "private": true,
4 | "version": "0.0.0",
5 | "type": "module",
6 | "scripts": {
7 | "dev": "vite",
8 | "build": "tsc && vite build",
9 | "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
10 | "preview": "vite preview"
11 | },
12 | "dependencies": {
13 | "@emotion/react": "^11.11.4",
14 | "@emotion/styled": "^11.11.5",
15 | "@mui/icons-material": "^5.15.20",
16 | "@mui/material": "^5.15.20",
17 | "axios": "^1.12.0",
18 | "install": "^0.13.0",
19 | "npm": "^10.8.1",
20 | "react": "^18.2.0",
21 | "react-dom": "^18.2.0",
22 | "react-router-dom": "^6.23.1",
23 | "styled-components": "^6.1.11"
24 | },
25 | "devDependencies": {
26 | "@types/node": "^20.14.11",
27 | "@types/react": "^18.2.66",
28 | "@types/react-dom": "^18.2.22",
29 | "@typescript-eslint/eslint-plugin": "^7.2.0",
30 | "@typescript-eslint/parser": "^7.2.0",
31 | "@vitejs/plugin-react": "^4.2.1",
32 | "eslint": "^8.57.0",
33 | "eslint-plugin-react-hooks": "^4.6.0",
34 | "eslint-plugin-react-refresh": "^0.4.6",
35 | "typescript": "^5.2.2",
36 | "vite": "^5.2.0"
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/FE/README.md:
--------------------------------------------------------------------------------
1 | # React + TypeScript + Vite
2 |
3 | This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
4 |
5 | Currently, two official plugins are available:
6 |
7 | - [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
8 | - [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
9 |
10 | ## Expanding the ESLint configuration
11 |
12 | If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
13 |
14 | - Configure the top-level `parserOptions` property like this:
15 |
16 | ```js
17 | export default {
18 | // other rules...
19 | parserOptions: {
20 | ecmaVersion: 'latest',
21 | sourceType: 'module',
22 | project: ['./tsconfig.json', './tsconfig.node.json'],
23 | tsconfigRootDir: __dirname,
24 | },
25 | }
26 | ```
27 |
28 | - Replace `plugin:@typescript-eslint/recommended` to `plugin:@typescript-eslint/recommended-type-checked` or `plugin:@typescript-eslint/strict-type-checked`
29 | - Optionally add `plugin:@typescript-eslint/stylistic-type-checked`
30 | - Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and add `plugin:react/recommended` & `plugin:react/jsx-runtime` to the `extends` list
31 |
--------------------------------------------------------------------------------
/FE/src/layouts/Chatbot/components/Message.tsx:
--------------------------------------------------------------------------------
1 | import { Box, Typography } from "@mui/material";
2 | import { MessageEntity } from "../Chatbot";
3 | import { AssistantAvatar, MsgWrapper, UserAvatar } from "../styles";
4 | import PersonIcon from "@mui/icons-material/Person";
5 |
6 | type MessageProps = {
7 | msg: MessageEntity;
8 | };
9 |
10 | const Message = (props: MessageProps) => {
11 | const { msg } = props;
12 | const isUser = msg.role === "User";
13 |
14 | const format_message = (message: string) => {
15 | console.log(message);
16 | const urlRegex = /]+>/g;
17 | return message
18 | .replace(urlRegex, (url) => {
19 | const cleanUrl = url.slice(1, -1);
20 | return `${cleanUrl}`;
21 | })
22 | .replace(/\n/g, "
");
23 | };
24 |
25 | return (
26 |
32 | {isUser && (
33 |
34 |
35 |
36 | )}
37 |
38 |
39 | {typeof msg.message != "string" ? (
40 | {msg.message}
41 | ) : (
42 |
45 | )}
46 |
47 |
48 | {!isUser && (
49 |
54 | )}
55 |
56 | );
57 | };
58 |
59 | export default Message;
60 |
--------------------------------------------------------------------------------
/FE/src/layouts/Sidebar/components/History.tsx:
--------------------------------------------------------------------------------
1 | import {
2 | ListItemText,
3 | ListItemIcon,
4 | ListItemButton,
5 | Divider,
6 | } from "@mui/material";
7 | import { useMemo } from "react";
8 | import List from "@mui/material/List";
9 | import ChatIcon from "@mui/icons-material/Chat";
10 | import { useNavigate, useParams } from "react-router-dom";
11 | import { CustomListItem } from "../styles";
12 | import { chatHistory } from "../../../data/data";
13 |
14 | const History = () => {
15 | const navigate = useNavigate();
16 | const { chatId } = useParams();
17 | const chatNumber = parseInt(chatId || "0");
18 |
19 | const historyElements = useMemo(
20 | () =>
21 | chatHistory.map((chat) => ({
22 | id: chat.id,
23 | text: chat.history[0].message.slice(0, 25) + "..",
24 | })),
25 | []
26 | );
27 |
28 | return (
29 | <>
30 | {historyElements && (
31 | <>
32 |
33 |
34 | {historyElements.map((item) => (
35 |
40 | navigate(`/dashboard/${item.id}`)}
42 | >
43 |
44 |
45 |
46 |
47 |
48 |
49 | ))}
50 |
51 | >
52 | )}
53 | >
54 | );
55 | };
56 |
57 | export default History;
58 |
--------------------------------------------------------------------------------
/FE/src/layouts/LandingBody/LandingBody.tsx:
--------------------------------------------------------------------------------
1 | import { Link } from "react-router-dom";
2 | import { ButtonsWrapper, Container, Typography } from "./styles";
3 |
4 | import { Button } from "@mui/material";
5 | import { useIsDarkTheme } from "../../contexts/useIsDarkTheme";
6 |
7 | const LandingBody = () => {
8 | const { isDarkTheme } = useIsDarkTheme();
9 | return (
10 |
11 |
12 | Welcome! to{" "}
13 |
18 | JenAi{" "}
19 |
20 |
21 |
22 | JenAi is your assistant today! JenAi was a GSoC'24 project by Jenkins.
23 | The project was to build a chatbot for specific Jenkins knowledge that
24 | can help users with their queries.
25 |
26 |
27 |
28 |
38 |
48 |
49 |
50 | );
51 | };
52 |
53 | export default LandingBody;
54 |
--------------------------------------------------------------------------------
/FE/src/layouts/Chatbot/components/Input.tsx:
--------------------------------------------------------------------------------
1 | import InputBase from "@mui/material/InputBase";
2 | import Divider from "@mui/material/Divider";
3 | import IconButton from "@mui/material/IconButton";
4 | import AddCircleOutlineIcon from "@mui/icons-material/AddCircleOutline";
5 | import SendIcon from "@mui/icons-material/Send";
6 | import { InputWrapper } from "../styles";
7 | import { useIsDarkTheme } from "../../../contexts/useIsDarkTheme";
8 |
9 | type InputProps = {
10 | setQuery: (query: string) => void;
11 | query: string;
12 | disabled: boolean;
13 | onSubmit: (e: React.FormEvent) => void;
14 | };
15 |
16 | const Input = (props: InputProps) => {
17 | const { setQuery, query, disabled, onSubmit } = props;
18 | const { isDarkTheme } = useIsDarkTheme();
19 |
20 | const handleKeyDown = (e: React.KeyboardEvent) => {
21 | if (e.key === "Enter") {
22 | e.preventDefault();
23 | onSubmit(e);
24 | }
25 | };
26 |
27 | return (
28 |
29 | setQuery(e.target.value)}
33 | onKeyDown={handleKeyDown}
34 | value={query}
35 | multiline
36 | fullWidth
37 | disabled={disabled}
38 | />
39 |
40 |
41 |
42 |
43 |
49 |
50 |
51 |
52 | );
53 | };
54 |
55 | export default Input;
56 |
--------------------------------------------------------------------------------
/FE/src/pages/Pages.tsx:
--------------------------------------------------------------------------------
1 | import { Routes, Route, Navigate } from "react-router-dom";
2 | import { CssBaseline, ThemeProvider } from "@mui/material";
3 | import { routes, RouteSchema } from "../routes";
4 | import { darkTheme, lightTheme } from "../theme";
5 | import { useIsDarkTheme } from "../contexts/useIsDarkTheme";
6 |
7 | import { createGlobalStyle } from "styled-components";
8 |
9 | const GlobalStyles = createGlobalStyle`
10 | /* width */
11 | ::-webkit-scrollbar {
12 | width: 8px;
13 | }
14 |
15 | /* Track */
16 | ::-webkit-scrollbar-track {
17 | box-shadow: inset 0 0 5px grey;
18 | border-radius: 10px;
19 | }
20 |
21 | /* Handle */
22 | ::-webkit-scrollbar-thumb {
23 | /* background: #f50057; */
24 | background: ${(props) => props.theme.palette.primary.main};
25 | border-radius: 10px;
26 | }
27 |
28 | /* Handle on hover */
29 | ::-webkit-scrollbar-thumb:hover {
30 | /* background: #f50057; */
31 | background: ${(props) => props.theme.palette.primary.main};
32 | }
33 | `;
34 |
35 | const Pages = () => {
36 | const { isDarkTheme } = useIsDarkTheme();
37 |
38 | const getRoutes = (allRoutes: RouteSchema[]) =>
39 | allRoutes.map((route: RouteSchema) => {
40 | if (route.route) {
41 | return (
42 |
47 | );
48 | }
49 |
50 | return null;
51 | });
52 |
53 | return (
54 |
55 |
56 |
57 |
58 | {getRoutes(routes)}
59 | } />
60 |
61 |
62 | );
63 | };
64 |
65 | export default Pages;
66 |
--------------------------------------------------------------------------------
/BE/chatbot/routes.py:
--------------------------------------------------------------------------------
1 | # chatbot/routes.py
2 | from flask import Blueprint, request, jsonify, Response, stream_with_context
3 | import logging
4 |
5 | from langchain_community.llms import CTransformers
6 | # from langchain.chains import LLMChain
7 | from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
8 | from langchain_core.prompts import PromptTemplate
9 |
10 | # Configure logging
11 | logging.basicConfig(filename='app.log', level=logging.ERROR,
12 | format='%(asctime)s %(levelname)s %(name)s %(message)s')
13 |
14 | chatbot_bp = Blueprint('chatbot', __name__)
15 |
16 | llm = CTransformers(model="nouralmulhem/Llama-2-7b-finetune-q8", model_file = 'model.bin', callbacks=[StreamingStdOutCallbackHandler()])
17 |
18 | template = """
19 | [INST] <>
20 | You are a helpful, respectful and honest assistant. Your answers are always very brief.
21 | {persona}
22 | Current conversation:
23 | {history}
24 | < >
25 | Human: {text}
26 | AI: [/INST]
27 | """
28 |
29 | prompt = PromptTemplate(template=template, input_variables=["text", "persona", "history"])
30 |
31 | # llm_chain = LLMChain(prompt=prompt, llm=llm)
32 | llm_chain = prompt | llm
33 |
34 |
35 | history = ''
36 |
37 | def process_request(query, persona):
38 | global history
39 | response = llm_chain.invoke({"text": query, "persona": persona, "history": history})
40 | history += f'Human: {query}\nAI: {response}\n'
41 | # print(f'{history}\n\n')
42 | return response
43 |
44 | @chatbot_bp.route('/chat', methods=['POST'])
45 | def chat():
46 | data = request.json
47 | query = data.get('text', '')
48 | persona = data.get('persona', '')
49 | try:
50 | prediction = process_request(query, persona)
51 | return jsonify({"prediction": prediction})
52 | except Exception as e:
53 | logging.error("Error occurred in /chat endpoint", exc_info=True)
54 | return jsonify({"prediction": "error"})
55 |
56 |
57 | @chatbot_bp.route('/test', methods=['GET'])
58 | def test():
59 | return jsonify({'test': 'OK'})
60 |
61 |
--------------------------------------------------------------------------------
/FE/src/layouts/Chatbot/styles.ts:
--------------------------------------------------------------------------------
1 | import { Avatar, Paper, styled } from "@mui/material";
2 | import { drawerWidth } from "../../constants";
3 | import { grey } from "@mui/material/colors";
4 |
5 | export const Container = styled(Paper)(({ theme }) => ({
6 | height: "100%",
7 | minWidth: "15%",
8 | width: "100%",
9 | margin: 30,
10 | backgroundColor: theme.palette.secondary.main,
11 | color: theme.palette.text.primary,
12 | display: "flex",
13 | flexDirection: "column",
14 | padding: 10,
15 | justifyContent: "center",
16 | alignItems: "center",
17 | gap: 20,
18 | }));
19 |
20 | export const Main = styled("main", {
21 | shouldForwardProp: (prop) => prop !== "open",
22 | })<{
23 | open?: boolean;
24 | }>(({ theme, open }) => ({
25 | flexGrow: 1,
26 | padding: theme.spacing(3),
27 | height: "85vh",
28 | width: "100%",
29 | transition: theme.transitions.create("margin", {
30 | easing: theme.transitions.easing.sharp,
31 | duration: theme.transitions.duration.leavingScreen,
32 | }),
33 | marginLeft: `-${drawerWidth}px`,
34 | ...(open && {
35 | transition: theme.transitions.create("margin", {
36 | easing: theme.transitions.easing.easeOut,
37 | duration: theme.transitions.duration.enteringScreen,
38 | }),
39 | marginLeft: 0,
40 | }),
41 | }));
42 |
43 | export const DrawerHeader = styled("div")(({ theme }) => ({
44 | display: "flex",
45 | alignItems: "center",
46 | padding: theme.spacing(0, 1),
47 | // necessary for content to be below app bar
48 | ...theme.mixins.toolbar,
49 | justifyContent: "flex-end",
50 | }));
51 |
52 | export const InputWrapper = styled(Paper)(() => ({
53 | padding: "2px 4px",
54 | display: "flex",
55 | alignItems: "center",
56 | width: "80%",
57 | }));
58 |
59 | export const MsgWrapper = styled(Paper)(({ theme }) => ({
60 | padding: 16,
61 | display: "flex",
62 | alignItems: "center",
63 | maxWidth: "70%",
64 | backgroundColor: theme.palette.mode === "dark" ? grey[900] : grey[300],
65 | }));
66 |
67 | export const UserAvatar = styled(Avatar)(({ theme }) => ({
68 | backgroundColor:
69 | theme.palette.mode === "dark"
70 | ? theme.palette.primary.main
71 | : theme.palette.secondary.light,
72 | }));
73 |
74 | export const AssistantAvatar = styled(Avatar)(() => ({
75 | height: 60,
76 | width: 40,
77 | }));
78 |
--------------------------------------------------------------------------------
/src/data preprocessing/preprocessing.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | from datasets import Dataset, concatenate_datasets
3 |
4 | def preprocess_data(file_name, questions_col, answers_col, max_sequence_length=11000, num_samples=None):
5 | """
6 | Preprocess the data to the desired format
7 | Args:
8 | file_name (str): The name of the file to be preprocessed
9 | questions_col (str): The name of the column containing the questions
10 | answers_col (str): The name of the column containing the answers
11 | max_sequence_length (int): The maximum length of the sequence
12 | num_samples (int): The number of samples to select from the dataset
13 | Returns:
14 | dataset: The preprocessed dataset contains the text column
15 | """
16 | raw_data = pd.read_csv(file_name)
17 |
18 | # Define condition to drop rows where answers_col has a code tag in
19 | condition = raw_data[answers_col].str.contains('')
20 | raw_data = raw_data[~condition]
21 |
22 | # Define condition to drop rows where questions_col has a code tag in
23 | condition = raw_data[questions_col].str.contains('')
24 | raw_data = raw_data[~condition]
25 |
26 | # Combine columns into the desired format
27 | df = pd.DataFrame(columns=['text'])
28 |
29 | df['text'] = raw_data.apply(
30 | lambda row: f"[INST] {row[questions_col].strip()} [/INST] {row[answers_col]} ",
31 | axis=1
32 | )
33 |
34 | df['text'] = df['text'].str.replace('\n', ' ')
35 |
36 | # Calculate the length of each row
37 | df['row_length'] = df['text'].apply(len)
38 |
39 | # Filter out rows with length more than max_sequence_length (i.e. 11000) to obtain just acceptable length of input
40 | df_filtered = df[df['row_length'] <= max_sequence_length]
41 |
42 | # drop un needed columns
43 | df_filtered = df_filtered.drop(['row_length'], axis=1)
44 |
45 | # convert to dataset and select num_samples (i.e. 1000) record for now (considering memory resource)
46 | dataset = Dataset.from_pandas(df_filtered)
47 |
48 | if num_samples and len(dataset) >= num_samples:
49 | dataset = dataset.select(range(num_samples))
50 |
51 | return dataset
52 |
53 | if __name__ == "__main__":
54 | max_sequence_length = 11000
55 | num_samples = None
56 |
57 | file_name = '../../datasets/QueryResultsUpdated.csv'
58 | questions_col = 'Question Body'
59 | answers_col = 'Answer Body'
60 | dataset1 = preprocess_data(file_name, questions_col, answers_col, max_sequence_length, num_samples)
61 |
62 | file_name = '../../datasets/Jenkins Docs QA.csv'
63 | questions_col = 'Question'
64 | answers_col = 'Answer'
65 | dataset2 = preprocess_data(file_name, questions_col, answers_col, max_sequence_length, num_samples)
66 |
67 | file_name = '../../datasets/Community Questions Refined.csv'
68 | questions_col = 'questions'
69 | answers_col = 'answers'
70 | dataset3 = preprocess_data(file_name, questions_col, answers_col, max_sequence_length, num_samples)
71 |
72 | dataset = concatenate_datasets([dataset1, dataset2, dataset3])
73 | # dataset = dataset2
74 | dataset.to_csv('../../datasets/final/final2.csv')
75 | print(len(dataset))
--------------------------------------------------------------------------------
/src/data collection/qa-article-to-qa-csv-pairs.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "import pandas as pd\n",
10 | "import csv"
11 | ]
12 | },
13 | {
14 | "cell_type": "code",
15 | "execution_count": 21,
16 | "metadata": {},
17 | "outputs": [
18 | {
19 | "name": "stdout",
20 | "output_type": "stream",
21 | "text": [
22 | "0 Q: What is Jenkins?\\nA: Jenkins is a self-cont...\n",
23 | "1 Q: What is the purpose of the tutorial \"Build ...\n",
24 | "2 **Q: Who is this LabVIEW CI/CD tutorial intend...\n",
25 | "3 Q: What is the concept behind using Jenkinsfil...\n",
26 | "4 Q: What does this tutorial demonstrate?\\nA: Th...\n",
27 | " ... \n",
28 | "91 Q: How can you refer to another project/job by...\n",
29 | "92 Q: What are the three flavors of remote access...\n",
30 | "93 Q: What is a Jenkins Agent?\\nA: A Jenkins Agen...\n",
31 | "94 Q: What is the importance of automating job de...\n",
32 | "95 Q: What is the purpose of file fingerprinting ...\n",
33 | "Name: 1, Length: 96, dtype: object\n"
34 | ]
35 | }
36 | ],
37 | "source": [
38 | "data = pd.read_excel(\"Jenkins Docs.xlsx\", header=None)\n",
39 | "\n",
40 | "second_column_values = data.iloc[:, 1]\n",
41 | "\n",
42 | "print(second_column_values)\n"
43 | ]
44 | },
45 | {
46 | "cell_type": "code",
47 | "execution_count": 53,
48 | "metadata": {},
49 | "outputs": [],
50 | "source": [
51 | "output_file = '../datasets/Jenkins Docs QA.csv'\n",
52 | "\n",
53 | "with open(output_file, \"w\", newline=\"\") as csvfile:\n",
54 | " fieldnames = [\"Question\", \"Answer\"]\n",
55 | " writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
56 | " writer.writeheader()\n",
57 | "\n",
58 | "for idx, text in enumerate(second_column_values):\n",
59 | " # Split the text into individual QA pairs\n",
60 | " # qas = text.strip().split(\"A:\")\n",
61 | "\n",
62 | " # Write the QA pairs to a CSV file\n",
63 | " with open(output_file, \"a\", newline=\"\") as csvfile:\n",
64 | " writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
65 | "\n",
66 | " dict = text.strip().split(\"Q:\")\n",
67 | " # print(len(dict[1:]), dict[1:])\n",
68 | " for item in dict[1:]:\n",
69 | " extracted = item.strip().split(\"A:\")\n",
70 | " if len(extracted) < 2:\n",
71 | " print(idx)\n",
72 | " print(item)\n",
73 | " print(extracted)\n",
74 | " else:\n",
75 | " question, answer = extracted \n",
76 | " \n",
77 | " if question.startswith(\"Q:\"):\n",
78 | " question = question[2:]\n",
79 | " writer.writerow({\"Question\": question.strip(), \"Answer\": answer.strip()})\n"
80 | ]
81 | }
82 | ],
83 | "metadata": {
84 | "kernelspec": {
85 | "display_name": "base",
86 | "language": "python",
87 | "name": "python3"
88 | },
89 | "language_info": {
90 | "codemirror_mode": {
91 | "name": "ipython",
92 | "version": 3
93 | },
94 | "file_extension": ".py",
95 | "mimetype": "text/x-python",
96 | "name": "python",
97 | "nbconvert_exporter": "python",
98 | "pygments_lexer": "ipython3",
99 | "version": "3.9.12"
100 | }
101 | },
102 | "nbformat": 4,
103 | "nbformat_minor": 2
104 | }
105 |
--------------------------------------------------------------------------------
/FE/src/layouts/Sidebar/Sidebar.tsx:
--------------------------------------------------------------------------------
1 | import {
2 | ListItemText,
3 | ListItemIcon,
4 | ListItemButton,
5 | Divider,
6 | Drawer,
7 | IconButton,
8 | ListItem,
9 | Typography,
10 | Box,
11 | TextField,
12 | } from "@mui/material";
13 | import { ReactNode } from "react";
14 | import ChevronLeftIcon from "@mui/icons-material/ChevronLeft";
15 | import List from "@mui/material/List";
16 | import DeleteIcon from "@mui/icons-material/Delete";
17 | import AddCircleOutlineIcon from "@mui/icons-material/AddCircleOutline";
18 | import PhonelinkSetupIcon from "@mui/icons-material/PhonelinkSetup";
19 | import { drawerWidth } from "../../constants";
20 | import { useNavigate } from "react-router-dom";
21 | import { DrawerHeader } from "./styles";
22 | import { usePersonalization } from "../../contexts/usePersonalization";
23 |
24 | type ControllersElement = {
25 | id: number;
26 | text: string;
27 | icon: ReactNode;
28 | };
29 |
30 | const ControllersElements: ControllersElement[] = [
31 | {
32 | id: 1,
33 | text: "Start New Chat",
34 | icon: ,
35 | },
36 | {
37 | id: 2,
38 | text: "Delete Chats",
39 | icon: ,
40 | },
41 | ];
42 |
43 | type SidebarProps = {
44 | open: boolean;
45 | handleDrawerClose: () => void;
46 | };
47 |
48 | const Sidebar = (props: SidebarProps) => {
49 | const { open, handleDrawerClose } = props;
50 | const navigate = useNavigate();
51 | const { handleUpdate } = usePersonalization();
52 |
53 | const handler = (id: number) => {
54 | switch (id) {
55 | case 1:
56 | return () => {
57 | navigate("/dashboard");
58 | };
59 | case 2:
60 | return () => {
61 | // sethistoryElements(undefined);
62 | };
63 | }
64 | };
65 |
66 | return (
67 |
80 |
81 |
82 |
83 |
84 | JenAI
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 | {/* */}
93 |
94 | {
111 | handleUpdate(e.target.value);
112 | }}
113 | />
114 |
115 |
116 | {ControllersElements.map((item) => (
117 |
118 |
119 | {item.icon}
120 |
121 |
122 |
123 | ))}
124 |
125 |
126 | );
127 | };
128 |
129 | export default Sidebar;
130 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 |
29 | # PyInstaller
30 | # Usually these files are written by a python script from a template
31 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
32 | *.manifest
33 | *.spec
34 |
35 | # Installer logs
36 | pip-log.txt
37 | pip-delete-this-directory.txt
38 |
39 | # Unit test / coverage reports
40 | htmlcov/
41 | .tox/
42 | .nox/
43 | .coverage
44 | .coverage.*
45 | .cache
46 | nosetests.xml
47 | coverage.xml
48 | *.cover
49 | *.py,cover
50 | .hypothesis/
51 | .pytest_cache/
52 | cover/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | .pybuilder/
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # IPython
82 | profile_default/
83 | ipython_config.py
84 |
85 | # pyenv
86 | # For a library or package, you might want to ignore these files since the code is
87 | # intended to run in multiple environments; otherwise, check them in:
88 | # .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # poetry
98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99 | # This is especially recommended for binary packages to ensure reproducibility, and is more
100 | # commonly ignored for libraries.
101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 |
104 | # pdm
105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | # in version control.
109 | # https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 |
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 |
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 |
119 | # SageMath parsed files
120 | *.sage.py
121 |
122 | # Environments
123 | # .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 |
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 |
135 | # Rope project settings
136 | .ropeproject
137 |
138 | # mkdocs documentation
139 | /site
140 |
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 |
146 | # Pyre type checker
147 | .pyre/
148 |
149 | # pytype static type analyzer
150 | .pytype/
151 |
152 | # Cython debug symbols
153 | cython_debug/
154 |
155 | # PyCharm
156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | # and can be added to the global gitignore or merged into this file. For a more nuclear
159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 | Llama-2-7b-q8/
162 | llama/
163 |
164 | *.bin
--------------------------------------------------------------------------------
/FE/src/data/data.ts:
--------------------------------------------------------------------------------
1 | export const chatHistory = [
2 | {
3 | id: 1,
4 | history: [
5 | {
6 | role: "User",
7 | message: `
8 | I've looked all over for this answer but cannot find it, as most of the answers are how to add JDK to Jenkins for usage within a test. I'm looking for how to ensure Jenkins itself is using OpenJDK 11 that I have installed and not the older JDK 8.
9 | I have checked the Jenkins.xml and it does point to OpenJDK 11 as well as the %java_home% and PATH in the system environment are pointed to the correct locations.
10 | When I check System Properties, I still see things related to Java 8 (java.home, java.specification.version, etc.)
11 | I have no idea where else it could be stored at to change this.
12 | `,
13 | },
14 | {
15 | role: "Assistant",
16 | message: `In your Jenkins UI select Manage Jenkins → Status Information → System Information (displayed as System Properties then; in Jenkins 2.289.3, at least) → search for java.runtime.version or others to look for hints what's possibly going wrong.`,
17 | },
18 | ],
19 | },
20 | {
21 | id: 2,
22 | history: [
23 | {
24 | role: "User",
25 | message: `
26 | pre-commit is a great tool and id like to incorporate it into jenkins pipelines just to make sure someone didn't try to skip it during committing to the repo.
27 | so my questions are:
28 |
29 | what are the best practices to initialize pre-commit? (just initialize it as i would in the repo? the website offers examples for some other CI integrations but not jenkins)
30 | how can i say to jenkins to stop the build if any of the tests performed by pre-commit fails? (exit code? which one would be success or fail?
31 | `,
32 | },
33 | {
34 | role: "Assistant",
35 | message: `
36 | as the docs indicate the best way to run pre-commit in a CI system is with pre-commit run, usually pre-commit run --all-files --show-diff-on-failure
37 | this will exit nonzero which should fail your pipeline as desired
38 |
39 | disclaimer: I'm the creator of pre-commit
40 | `,
41 | },
42 | ],
43 | },
44 | {
45 | id: 3,
46 | history: [
47 | {
48 | role: "User",
49 | message: `
50 | what is Jenkins?
51 | `,
52 | },
53 | {
54 | role: "Assistant",
55 | message: `
56 | Jenkins is an open-source automation server widely used for continuous integration and continuous delivery (CI/CD). It helps automate the parts of software development related to building, testing, and deploying, facilitating continuous integration and continuous delivery.
57 |
58 | Here are some key features of Jenkins:
59 |
60 | 1. **Extensibility**: Jenkins has a large number of plugins available, which allows it to integrate with many other tools and services. This makes it highly customizable to suit various project needs.
61 |
62 | 2. **Distributed Builds**: Jenkins can distribute build/test loads to multiple machines, making it highly scalable.
63 |
64 | 3. **Easy Installation and Configuration**: Jenkins can be installed easily on various platforms and can be configured via its web interface.
65 |
66 | 4. **Pipeline as Code**: Jenkins supports defining pipelines as code using a domain-specific language (DSL) and storing them in a version control system (like Git). This is known as "Jenkins Pipeline" or "Pipeline as Code".
67 |
68 | 5. **Active Community**: Being open-source, Jenkins has a large, active community contributing to its development and providing support.
69 |
70 | 6. **Extensive Plugin Ecosystem**: Jenkins' functionality can be extended through plugins, which cover a wide range of purposes from source code management, build tools, and testing, to notifications and user interfaces.
71 |
72 | Jenkins is used by many organizations to automate their development processes, ensuring faster and more reliable software delivery.
73 | `,
74 | },
75 | ],
76 | },
77 | ];
78 |
--------------------------------------------------------------------------------
/FE/src/layouts/Chatbot/Chatbot.tsx:
--------------------------------------------------------------------------------
1 | import { useParams } from "react-router-dom";
2 | import { Container, DrawerHeader, Main } from "./styles";
3 | import { Box, useTheme } from "@mui/material";
4 | import { ReactNode, useEffect, useState } from "react";
5 |
6 | import { chatHistory } from "../../data/data";
7 | import Input from "./components/Input";
8 | import Message from "./components/Message";
9 | import { sendQuery } from "./server";
10 | import { usePersonalization } from "../../contexts/usePersonalization";
11 |
12 | type ChatbotProps = {
13 | open: boolean;
14 | };
15 |
16 | export type MessageEntity = {
17 | role: string;
18 | message: ReactNode;
19 | };
20 |
21 | const Chatbot = (props: ChatbotProps) => {
22 | const { open } = props;
23 | const { chatId } = useParams();
24 | const theme = useTheme();
25 | const chatNumber = parseInt(chatId || "0");
26 | const { text: personalization } = usePersonalization();
27 |
28 | const [conversation, setConversation] = useState([]);
29 | const [query, setQuery] = useState("");
30 | const [disabled, setDisabled] = useState(false);
31 | const [loading, setLoading] = useState(false);
32 |
33 | const onSubmit = (e: React.FormEvent) => {
34 | e.preventDefault();
35 | setDisabled(true);
36 |
37 | setConversation((prevConversation) => [
38 | ...prevConversation,
39 | { role: "User", message: query },
40 | ]);
41 | setQuery("");
42 | setLoading(true);
43 |
44 | sendQuery(query, personalization)
45 | .then((message) => {
46 | setTimeout(() => {
47 | setConversation((prevConversation) => [
48 | ...prevConversation,
49 | { role: "Assistant", message: message },
50 | ]);
51 | setDisabled(false);
52 | setLoading(false);
53 | }, 2000);
54 | })
55 | .catch((error) => {
56 | console.error("Error sending query:", error);
57 | setLoading(false);
58 | setDisabled(false);
59 | });
60 |
61 | // setTimeout(() => {
62 | // setConversation((prevConversation) => [
63 | // ...prevConversation,
64 | // { role: "Assistant", message: "Response" },
65 | // ]);
66 | // setLoading(false);
67 | // setDisabled(false);
68 | // }, 2000);
69 | };
70 |
71 | useEffect(() => {
72 | if (chatNumber >= 1) {
73 | setConversation(
74 | chatHistory.find((chat) => chat["id"] === chatNumber)?.history || []
75 | );
76 | } else {
77 | setConversation([]);
78 | }
79 | }, [chatNumber]);
80 |
81 | return (
82 |
83 |
84 |
85 |
86 |
96 | {conversation.map((msg, index) => (
97 |
98 | ))}
99 | {loading && (
100 |
114 | ),
115 | }}
116 | />
117 | )}
118 |
119 |
125 |
126 |
127 |
128 | );
129 | };
130 |
131 | export default Chatbot;
132 |
--------------------------------------------------------------------------------
/FE/src/layouts/Header/Header.tsx:
--------------------------------------------------------------------------------
1 | import {
2 | Box,
3 | Button,
4 | FormControlLabel,
5 | FormGroup,
6 | IconButton,
7 | styled,
8 | Switch,
9 | Toolbar,
10 | Typography,
11 | useTheme,
12 | } from "@mui/material";
13 | import { AppBar } from "./styles";
14 | import { Link, useLocation } from "react-router-dom";
15 | import MenuIcon from "@mui/icons-material/Menu";
16 | import { useMemo } from "react";
17 | import { useIsDarkTheme } from "../../contexts/useIsDarkTheme";
18 |
19 | type HeaderProps = {
20 | open?: boolean;
21 | handleDrawerOpen?: () => void;
22 | };
23 |
24 | interface MaterialUISwitchProps {
25 | isDarkTheme: boolean;
26 | }
27 |
28 | const MaterialUISwitch = styled(Switch)(
29 | ({ theme, isDarkTheme }) => ({
30 | width: 62,
31 | height: 34,
32 | padding: 7,
33 | "& .MuiSwitch-switchBase": {
34 | margin: 1,
35 | padding: 0,
36 | transform: "translateX(6px)",
37 | "&.Mui-checked": {
38 | color: "#fff",
39 | transform: "translateX(22px)",
40 | "& .MuiSwitch-thumb:before": {
41 | backgroundImage: `url('data:image/svg+xml;utf8,')`,
44 | },
45 | "& + .MuiSwitch-track": {
46 | opacity: 1,
47 | backgroundColor: isDarkTheme ? "#8796A5" : "#aab4be",
48 | },
49 | },
50 | },
51 | "& .MuiSwitch-thumb": {
52 | backgroundColor: isDarkTheme
53 | ? theme.palette.primary.main
54 | : theme.palette.primary.dark,
55 | width: 32,
56 | height: 32,
57 | "&::before": {
58 | content: "''",
59 | position: "absolute",
60 | width: "100%",
61 | height: "100%",
62 | left: 0,
63 | top: 0,
64 | backgroundRepeat: "no-repeat",
65 | backgroundPosition: "center",
66 | backgroundImage: `url('data:image/svg+xml;utf8,')`,
69 | },
70 | },
71 | "& .MuiSwitch-track": {
72 | opacity: 1,
73 | backgroundColor: isDarkTheme ? "#8796A5" : "#aab4be",
74 | borderRadius: 20 / 2,
75 | },
76 | })
77 | );
78 |
79 | const Header = (props: HeaderProps) => {
80 | const { open, handleDrawerOpen } = props;
81 | const theme = useTheme();
82 | const { pathname } = useLocation();
83 | const { isDarkTheme, handleThemeChange } = useIsDarkTheme();
84 |
85 | const isLandinPage = useMemo(() => pathname === "/landing", [pathname]);
86 |
87 | return (
88 |
89 |
90 |
91 |
92 | {!isLandinPage && (
93 |
99 |
100 |
101 | )}
102 |
110 |
111 | Jenkins Chatbot
112 |
113 |
114 |
115 |
124 |
137 |
138 |
141 | }
142 | label={isDarkTheme ? "Light Mode" : "Dark Mode"}
143 | onChange={handleThemeChange}
144 | sx={{ color: "white" }}
145 | />
146 |
147 |
148 |
149 |
150 |
151 | );
152 | };
153 |
154 | export default Header;
155 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Enhancing-LLM-with-Jenkins-Knowledge
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 | ## Overview
18 |
19 |
20 | - Built using Python.
21 | - This Project is from Google Summer of code 2024
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 | ## Setting up env
34 |
35 | you may need to update the environment variables set in `BE/.env` and `FE/.env`
36 |
37 | ### Frontend env setup
38 |
39 | - reach to `.env` file in `FE/` directory, you will find the url setup by default to localhost
40 |
41 | ```sh
42 | VITE_SERVER_URL = http://127.0.0.1:5000/
43 | ```
44 |
45 | ### Backend env setup
46 |
47 | - reach to `.env` file in `BE/` directory, you will find also both `HOST` and `PORT` which are configured to localhost be default
48 |
49 | ```sh
50 | FLASK_RUN_HOST = 0.0.0.0
51 | FLASK_RUN_PORT = 5000
52 | ```
53 |
54 |
55 |
56 |
57 |
58 |
59 | ## How To Run
60 |
61 | Open a new terminal in the project directory
62 |
63 | ### Frontend server setup
64 |
65 | - Need to install [Node](https://nodejs.org/en/download/package-manager) first
66 | - Install all required packages
67 |
68 | ```sh
69 | cd ./FE
70 | npm install
71 | ```
72 | - Start the server
73 |
74 | ```sh
75 | npm run dev
76 | ```
77 | - You will get a message that the server is running at http://localhost:5173/
78 |
79 | ### Backend server setup
80 |
81 | - Install the needed packages.
82 |
83 | ```sh
84 | cd ./BE
85 | python3 -m venv .
86 | source ./bin/activate
87 | pip install -r ./requirements.txt
88 | ```
89 |
90 | - Start the server
91 |
92 | ```sh
93 | python app.py
94 | ```
95 |
96 | - note that if you are running the BE server for the first time so it will download the model locally on your machine and it is about 6GB, notice: this is for the first time you are running this only
97 |
98 |
99 |
100 |
101 |
102 | ## Fine-Tune your version
103 |
104 | You can fine-tune your own version and get it uploaded on hugging face using the following steps
105 |
106 | - we fine-tune llama2 using colab free resources of T4 GPU with 16 GB VRAM
107 | - we provided `./src/Fine-Tuning.ipynb`
108 | - we clone our repository to access the dataset provided for training
109 | ```sh
110 | git clone https://github.com/nouralmulhem/Enhancing-LLM-with-Jenkins-Knowledge.git
111 | ```
112 |
113 | - drive is used to store the checkpoints just to ensure its persistance in case of colab enviornment crashes
114 |
115 | you can edit the path to drive you want to save the model in by editting `new_model_path` variable
116 |
117 | - you also can set the number of epochs you would like to use to fine-tune the model by updating `num_train_epochs` variable
118 |
119 | - after getting done with fine-tuning the model you can access `./src/Upload_Model.ipynb` to merge lora weights with the model and upload your own model on hugging face and start using it
120 |
121 | - at this stage you need to update `new_model_path` variable to the correct path on your drive
122 |
123 | - as a final step you need to update `repo_id` variable to match your repo on hugging face
124 |
125 | VOILA! you got your own model
126 |
127 |
128 |
129 |
130 |
131 | ## Convert fine-tuned to GGML
132 |
133 | ### CPU model
134 |
135 | You can load this full model onto the GPU and run it like you would any other hugging face model, but we are here to take it to the next level of running this model on the CPU.
136 |
137 | we are using llama.cpp, so first of all we need to clone the repo
138 |
139 | ```sh
140 | git clone https://github.com/ggerganov/llama.cpp.git
141 | ```
142 |
143 | Llama.cpp has a script called `convert_hf_to_gguf.py` that is used to convert models to the binary GGML format that can be loaded and run on CPU.
144 |
145 | ```sh
146 | python convert_hf_to_gguf.py path/to/fine-tuned/model/ --outtype f16 --outfile path/to/binary/model.bin
147 | ```
148 |
149 | This should output a 13GB binary file at the specified `path/to/binary/model.bin` that is ready to run on CPU with the same code that we started with!
150 |
151 | ### Quantization
152 |
153 | Part of the appeal of the GGML library is being able to quantize this 13GB model into smaller models that can be run even faster. There is a tool called quantize in the Llama.cpp repo that can be used to convert the model to different quantization levels.
154 |
155 | First you need to build the tools in the Llama.cpp repository.
156 |
157 | ```sh
158 | cd llama.cpp
159 | cmake -B build
160 | cmake --build build --config Release
161 | ```
162 |
163 | This will create the tools in the bin directory. You can now use the quantize tool to shrink our model to q8_0 by running:
164 |
165 | ```sh
166 | cd build/bin/release
167 | ./llama-quantize.exe path/to/binary/model.bin path/to/binary/merged-q8_0.bin q8_0
168 | ```
169 | Now we have a 6.7 GB model at path/to/binary/merged-q8_0.bin
170 |
171 | To upload the local quantized model on huggingface
172 | ```sh
173 | huggingface-cli upload username/repo_id path/to/binary/quantized/model.bin model.bin
174 | ```
175 |
176 |
177 |
178 |
179 |
180 |
181 | ## Contributors
182 |
183 |
184 |
185 | 
Nour Almulhem
186 |
187 |
188 |
189 |
190 |
191 |
192 | ## 🔒 License
193 |
194 | > **Note**: This software is licensed under MIT License, See [License](https://github.com/nouralmulhem/Enhancing-LLM-with-Jenkins-Knowledge/blob/main/LICENSE) for more information ©nouralmulhem.
195 |
--------------------------------------------------------------------------------
/src/data collection/parse-jenkins-community.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 35,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "import requests\n",
10 | "import numpy as np\n",
11 | "from utils import *"
12 | ]
13 | },
14 | {
15 | "cell_type": "code",
16 | "execution_count": 36,
17 | "metadata": {},
18 | "outputs": [],
19 | "source": [
20 | "topics_list = []\n",
21 | "number_of_pages = 100\n",
22 | "URLs = [\n",
23 | " \"https://community.jenkins.io/c/using-jenkins/support/8.json\"\n",
24 | "]\n",
25 | "\n",
26 | "for page in URLs:\n",
27 | " \n",
28 | " for page_num in range(number_of_pages): \n",
29 | " \n",
30 | " # URL to make the GET request to\n",
31 | " url = f\"{page}?page={page_num}\"\n",
32 | "\n",
33 | " # Make the GET request\n",
34 | " response = requests.get(url)\n",
35 | "\n",
36 | " if response.status_code == 200:\n",
37 | " # Parse the JSON response into a Python dictionary\n",
38 | " json_response = response.json()\n",
39 | " \n",
40 | " \n",
41 | " # Extract posts from the JSON object\n",
42 | " topics = json_response[\"topic_list\"][\"topics\"]\n",
43 | " # print(page_num, len(topics))\n",
44 | " \n",
45 | " # Find the post with accepted_answer = true\n",
46 | " for topic in topics:\n",
47 | " if topic.get(\"has_accepted_answer\"):\n",
48 | " topics_list.append(topic)\n",
49 | "\n",
50 | " else:\n",
51 | " # Print an error message if the request was not successful\n",
52 | " print(\"Error:\", response.status_code)\n"
53 | ]
54 | },
55 | {
56 | "cell_type": "code",
57 | "execution_count": 37,
58 | "metadata": {},
59 | "outputs": [
60 | {
61 | "name": "stdout",
62 | "output_type": "stream",
63 | "text": [
64 | "344\n"
65 | ]
66 | }
67 | ],
68 | "source": [
69 | "print(len(topics_list))"
70 | ]
71 | },
72 | {
73 | "cell_type": "code",
74 | "execution_count": 38,
75 | "metadata": {},
76 | "outputs": [],
77 | "source": [
78 | "# Get the topics IDs\n",
79 | "topics_ids = [topic[\"id\"] for topic in topics_list]"
80 | ]
81 | },
82 | {
83 | "cell_type": "code",
84 | "execution_count": 39,
85 | "metadata": {},
86 | "outputs": [
87 | {
88 | "name": "stdout",
89 | "output_type": "stream",
90 | "text": [
91 | "No accepted answer found for topic 11272\n"
92 | ]
93 | }
94 | ],
95 | "source": [
96 | "accepted_answers = []\n",
97 | "original_questions = []\n",
98 | "\n",
99 | "for topic_id in topics_ids:\n",
100 | " \n",
101 | " # URL to make the GET request to\n",
102 | " url = f\"https://community.jenkins.io/t/{topic_id}.json?track_visit=true&forceLoad=true\"\n",
103 | "\n",
104 | " # Make the GET request\n",
105 | " response = requests.get(url)\n",
106 | "\n",
107 | " if response.status_code == 200:\n",
108 | " # Parse the JSON response into a Python dictionary\n",
109 | " json_response = response.json()\n",
110 | " \n",
111 | " \n",
112 | " # Extract posts from the JSON object\n",
113 | " posts = json_response[\"post_stream\"][\"posts\"]\n",
114 | " # print(posts)\n",
115 | " \n",
116 | " # Find the post with accepted_answer = true\n",
117 | " for post in posts:\n",
118 | " if post.get(\"accepted_answer\"):\n",
119 | " original_questions.append(posts[0]['cooked'])\n",
120 | " accepted_answers.append(post['cooked'])\n",
121 | " break\n",
122 | " else:\n",
123 | " print(f\"No accepted answer found for topic {topic_id}\")\n",
124 | "\n",
125 | " else:\n",
126 | " # Print an error message if the request was not successful\n",
127 | " print(\"Error:\", response.status_code)\n"
128 | ]
129 | },
130 | {
131 | "cell_type": "code",
132 | "execution_count": 40,
133 | "metadata": {},
134 | "outputs": [
135 | {
136 | "name": "stdout",
137 | "output_type": "stream",
138 | "text": [
139 | "343 343\n"
140 | ]
141 | }
142 | ],
143 | "source": [
144 | "print(len(original_questions), len(accepted_answers))"
145 | ]
146 | },
147 | {
148 | "cell_type": "code",
149 | "execution_count": 41,
150 | "metadata": {},
151 | "outputs": [
152 | {
153 | "name": "stdout",
154 | "output_type": "stream",
155 | "text": [
156 | "Jenkins Controller Version: 2.401.2 LTS
\n",
157 | "We have a Jenkins controller installed and configured using the official Jenkins Helm chart on an EKS cluster, with all resource provisioning managed by Terraform.
\n",
158 | "Currently, it is functioning well and is accessible at https://jenkins-utility.example.com/ using our LDAP credentials. All agent nodes connect to same jenkins endpoint.
\n",
159 | "However, when the Jenkins controller is restarted either via the URL using safe restart or due to the pod being killed/restarted, I can still access Jenkins using the same URL. Despite this, all agent nodes get disconnected because the Jenkins location URL resets to localhost:8080 instead of https://jenkins-utility.example.com/. Please see the attached image for reference.
\n",
160 | "Below is the Jenkins Configuration as Code (CasC) configuration we are using:
\n",
161 | "jenkins:\n",
162 | " authorizationStrategy:\n",
163 | " projectMatrix:\n",
164 | " entries:\n",
165 | " - group:\n",
166 | " name: \"authenticated\"\n",
167 | " permissions:\n",
168 | " - \"Agent/Configure\"\n",
169 | " - \"Agent/Connect\"\n",
170 | " - \"Agent/Create\"\n",
171 | " - \"Agent/Delete\"\n",
172 | " - \"Agent/Disconnect\"\n",
173 | " - \"Credentials/View\"\n",
174 | " - \"Job/Build\"\n",
175 | " - \"Job/Cancel\"\n",
176 | " - \"Job/Configure\"\n",
177 | " - \"Job/Create\"\n",
178 | " - \"Job/Delete\"\n",
179 | " - \"Job/Discover\"\n",
180 | " - \"Job/Move\"\n",
181 | " - \"Job/Read\"\n",
182 | " - \"Job/Workspace\"\n",
183 | " - \"Overall/Read\"\n",
184 | " - \"Overall/SystemRead\"\n",
185 | " - \"Run/Replay\"\n",
186 | " - \"View/Configure\"\n",
187 | " - \"View/Create\"\n",
188 | " - \"View/Delete\"\n",
189 | " - \"View/Read\"\n",
190 | " - group:\n",
191 | " name: \"jenkins-admins\"\n",
192 | " permissions:\n",
193 | " - \"Overall/Administer\"\n",
194 | " - user:\n",
195 | " name: \"anonymous\"\n",
196 | " permissions:\n",
197 | " - \"Job/Discover\"\n",
198 | " - \"Job/ViewStatus\"\n",
199 | " - \"Overall/Read\"\n",
200 | " - user:\n",
201 | " name: \"altif@example.com\"\n",
202 | " permissions:\n",
203 | " - \"Overall/Administer\"\n",
204 | " securityRealm:\n",
205 | " activeDirectory:\n",
206 | " bindPassword: \"xxxxxxxx\"\n",
207 | " cache:\n",
208 | " size: 1000\n",
209 | " ttl: 3600\n",
210 | " customDomain: true\n",
211 | " domains:\n",
212 | " - bindPassword: \"xxxxxxxx\"\n",
213 | " name: \"cor.example.com\"\n",
214 | " servers: \"ldap.example.com:3268\"\n",
215 | " tlsConfiguration: TRUST_ALL_CERTIFICATES\n",
216 | " groupLookupStrategy: TOKENGROUPS\n",
217 | " removeIrrelevantGroups: false\n",
218 | " requireTLS: false\n",
219 | " startTls: true\n",
220 | " disableRememberMe: false\n",
221 | " mode: NORMAL\n",
222 | " numExecutors: 2\n",
223 | " labelString: \"controller\"\n",
224 | " projectNamingStrategy: \"standard\"\n",
225 | " markupFormatter:\n",
226 | " plainText\n",
227 | " clouds:\n",
228 | " - kubernetes:\n",
229 | " containerCapStr: \"10\"\n",
230 | " defaultsProviderTemplate: \"\"\n",
231 | " connectTimeout: \"5\"\n",
232 | " readTimeout: \"15\"\n",
233 | " jenkinsUrl: \"http://jenkins-utility-prod.jenkins-utility-prod.svc.cluster.local:8080\"\n",
234 | " jenkinsTunnel: \"jenkins-utility-prod-agent.jenkins-utility-prod.svc.cluster.local:50000\"\n",
235 | " maxRequestsPerHostStr: \"32\"\n",
236 | " name: \"kubernetes\"\n",
237 | " namespace: \"jenkins-utility-prod\"\n",
238 | " serverUrl: \"https://kubernetes.default\"\n",
239 | " crumbIssuer:\n",
240 | " standard:\n",
241 | " excludeClientIPFromCrumb: false\n",
242 | "security:\n",
243 | " apiToken:\n",
244 | " creationOfLegacyTokenEnabled: false\n",
245 | " tokenGenerationOnCreationEnabled: false\n",
246 | " usageStatisticsEnabled: true\n",
247 | "unclassified:\n",
248 | " location:\n",
249 | " adminAddress: devops@example.com\n",
250 | " url: https://jenkins-utility.example.com/\n",
251 | "
\n",
252 | "I have spent many hours troubleshooting this issue but have not found a solution yet. Any assistance or references to resolve this issue would be greatly appreciated.
\n",
253 | "Thanks.
\n",
254 | "
Thank you @mawinter69 @poddingue for your prompt responses and willingness to assist, much appreciated!
\n",
255 | "Although, the initial suggestions didn’t resolve my issue, I continued my investigation. After exploring and digging JENKINS_HOME i.e. /var/jenkins_home, I noticed the init.groovy.d directory, I discovered groovy script named base.groovy that was resetting the Jenkins Location URL to localhost:8080. By modifying this script, I was able to set the correct base URL, and now everything works as expected.
\n",
256 | "Thank you all for your support!
\n"
257 | ]
258 | }
259 | ],
260 | "source": [
261 | "print(original_questions[0], accepted_answers[0])"
262 | ]
263 | },
264 | {
265 | "cell_type": "code",
266 | "execution_count": 33,
267 | "metadata": {},
268 | "outputs": [],
269 | "source": [
270 | "combined_arrays = np.column_stack((original_questions, accepted_answers))\n",
271 | "\n",
272 | "df = pd.DataFrame(combined_arrays, columns=['questions', 'answers'])\n",
273 | "\n",
274 | "df['questions'] = df['questions'].apply(remove_html_tags)\n",
275 | "df['answers'] = df['answers'].apply(remove_html_tags)"
276 | ]
277 | },
278 | {
279 | "cell_type": "code",
280 | "execution_count": 34,
281 | "metadata": {},
282 | "outputs": [],
283 | "source": [
284 | "df.to_csv('../datasets/Community Questions Refined.csv', index=False, encoding='utf-8', date_format=str)"
285 | ]
286 | },
287 | {
288 | "cell_type": "code",
289 | "execution_count": 18,
290 | "metadata": {},
291 | "outputs": [
292 | {
293 | "name": "stdout",
294 | "output_type": "stream",
295 | "text": [
296 | "CSV file with arrays saved as 'special_characters.csv'\n"
297 | ]
298 | }
299 | ],
300 | "source": [
301 | "# import csv\n",
302 | "\n",
303 | "# rows = list(zip(original_questions, accepted_answers))\n",
304 | "\n",
305 | "# with open('Community Questions.csv', 'w', newline='', encoding='utf-8') as csvfile:\n",
306 | "# writer = csv.writer(csvfile)\n",
307 | "# writer.writerow(['Question', 'Answer']) # Write header\n",
308 | " \n",
309 | "# for row in rows:\n",
310 | "# writer.writerow(row)\n",
311 | "\n",
312 | "# print(\"CSV file with arrays saved as 'special_characters.csv'\")\n"
313 | ]
314 | },
315 | {
316 | "cell_type": "code",
317 | "execution_count": 53,
318 | "metadata": {},
319 | "outputs": [],
320 | "source": [
321 | "# import json\n",
322 | "# # Write posts data to a JSON file\n",
323 | "# with open(\"show.json\", \"w\") as json_file:\n",
324 | "# json.dump(posts, json_file, indent=4)"
325 | ]
326 | },
327 | {
328 | "cell_type": "code",
329 | "execution_count": 54,
330 | "metadata": {},
331 | "outputs": [],
332 | "source": [
333 | "# accepted_answers[0]"
334 | ]
335 | },
336 | {
337 | "cell_type": "code",
338 | "execution_count": null,
339 | "metadata": {},
340 | "outputs": [],
341 | "source": []
342 | }
343 | ],
344 | "metadata": {
345 | "kernelspec": {
346 | "display_name": "base",
347 | "language": "python",
348 | "name": "python3"
349 | },
350 | "language_info": {
351 | "codemirror_mode": {
352 | "name": "ipython",
353 | "version": 3
354 | },
355 | "file_extension": ".py",
356 | "mimetype": "text/x-python",
357 | "name": "python",
358 | "nbconvert_exporter": "python",
359 | "pygments_lexer": "ipython3",
360 | "version": "3.11.3"
361 | }
362 | },
363 | "nbformat": 4,
364 | "nbformat_minor": 2
365 | }
366 |
--------------------------------------------------------------------------------
/src/data collection/refine-html-tags.ipynb:
--------------------------------------------------------------------------------
1 | {"cells":[{"cell_type":"code","execution_count":null,"metadata":{"id":"JxCCtmYCKXuO"},"outputs":[],"source":["from utils import *"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":81},"executionInfo":{"elapsed":12,"status":"ok","timestamp":1717154418699,"user":{"displayName":"Nour Ziad Almulhem","userId":"13402313266978952785"},"user_tz":-180},"id":"SwPtcV_cINsT","outputId":"673b5589-94c5-453e-b477-0bd94213affc"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"summary":"{\n \"name\": \"df\",\n \"rows\": 341,\n \"fields\": [\n {\n \"column\": \"questions\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 341,\n \"samples\": [\n \"All my jobs are configured by Job DSL. I connect to bitbucket via Bitbucket Server Integration. Unfortunately, I cannot find how to set \\u201cBitbucket webhook trigger\\u201d option from Job DSL:
\\n
\\nDo you have any suggestions on how to do that?
\\nHere you can find example DSL code:
\\nmultibranchPipelineJob(\\\"Repo name\\\") {\\n branchSources {\\n branchSource {\\n source {\\n BbS {\\n id(\\\"all-branches\\\")\\n credentialsId(\\\"ci.user_password\\\")\\n sshCredentialsId(\\\"ci.user\\\")\\n serverId(\\\"XXXX\\\")\\n projectName(\\\"YYYYYY\\\")\\n repositoryName(repository)\\n mirrorName(\\\"\\\")\\n traits {\\n gitBranchDiscovery()\\n headWildcardFilter {\\n includes('*')\\n excludes(\\\"\\\")\\n }\\n }\\n }\\n }\\n }\\n }\\n\\n triggers {\\n //any trigger or something?\\n }\\n}\\n
\",\n \"Hi, I have created a docker image using this script:
\\nFROM centos:7\\n\\nRUN yum install -y \\\\\\n wget \\\\\\n git \\\\\\n make \\\\\\n tar \\\\\\n centos-release-scl \\\\\\n sudo\\n\\nRUN mkdir -p /root/.ssh && \\\\\\ncd /root/.ssh && \\\\\\nwget http://192.168.111.64/env/auto-keys.tar && \\\\\\ntar -xvf auto-keys.tar\\n\\nRUN cd /root && wget https://repo.anaconda.com/archive/Anaconda3-2020.02-Linux-x86_64.sh\\n\\nRUN cd /root && sudo chmod 777 Anaconda3-2020.02-Linux-x86_64.sh && \\\\\\nsudo ./Anaconda3-2020.02-Linux-x86_64.sh -b -p /root/anaconda3 && \\\\\\nsudo ln -fs /root/anaconda3/bin/python /usr/bin/python3 && \\\\\\nsudo ln -fs /root/anaconda3/bin/python /bin/python3 && \\\\\\nrm -fr Anaconda3-2020.02-Linux-x86_64.sh\\n\\nRUN cd /usr/local && \\\\\\nwget http://192.168.111.64/env/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz && \\\\\\ntar -xvf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz && \\\\\\nrm -rf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz\\n\\nRUN cd /usr/local && \\\\\\nwget http://192.168.111.64/env/gcc-arm-8.3-2019.03-x86_64-aarch64-elf.tar.xz && \\\\\\ntar -xvf gcc-arm-8.3-2019.03-x86_64-aarch64-elf.tar.xz && \\\\\\nrm -rf gcc-arm-8.3-2019.03-x86_64-aarch64-elf.tar.xz\\n\\nRUN yum install -y \\\\\\n sshpass \\\\\\n devtoolset-8\\n\\nRUN yum install -y openssl\\n\\nRUN echo 'source /opt/rh/devtoolset-8/enable' | sudo tee /etc/profile.d/gcc-version.sh && source /etc/profile.d/gcc-version.sh\\n# RUN rm -rf /usr/bin/gcc\\n\\n# RUN sudo ln -s /opt/rh/devtoolset-8/root/bin/gcc /usr/bin/gcc\\n\\nENV PATH /usr/local/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu/bin:${PATH}\\nENV PATH /usr/local/gcc-arm-8.3-2019.03-x86_64-aarch64-elf/bin:${PATH}\\n# ENV PATH /opt/rh/devtoolset-8/root/bin/gcc:${PATH}\\n\\n# RUN cd ~ && sed -i '$aexport PATH=$PATH:/usr/local/sbin:/usr/sbin' .bashrc && source .bashrc\\n
\\nThis is my pipeline:
\\nnode('192.168.111.134'){\\n def myImage\\n myImage = docker.image 'prbuild:v7'\\n stage('inside'){\\n \\n myImage.inside{\\n sh '''\\n cd /root/${project}\\n git pull\\n git branch\\n ./quince_x86-64_wout_build_b58R.sh\\n\\n '''\\n }\\n }\\n\\n}\\n
\\nThe error I came across
\\n10:45:10 /bin/sh: g++: command not found\\n10:45:10 /bin/sh: g++: command not found\\n10:45:10 /bin/sh: g++: command not found\\n10:45:10 /bin/sh: g++: command not found\\n10:45:10 /bin/sh: g++: command not found\\n10:45:10 /bin/sh: g++: command not found\\n10:45:10 /bin/sh: g++: command not found\\n10:45:10 /bin/sh: g++: command not found\\n10:45:10 /bin/sh: g++: command not found\\n
\\nThe question is I can run these scripts on my local container correctly but it show /bin/sh: g++: command not found when I am running the same script in Jenkins
\",\n \"Hi,
\\nmy shared library needs to parse the job config.xml to get the configured Jenkinsfile path of a multibranch pipeline.
\\nwith Jenkins 2.375.4.2 this works fine
\\ndef call() {\\n withCredentials([usernamePassword(credentialsId: 'xxx', passwordVariable: 'pw', usernameVariable: 'user')]) {\\n genericSh('curl -u ' + user + ':' + pw + \\\" ${JOB_URL + 'config.xml'} -O\\\")\\n }\\n jobconfig = readFile \\\"${env.WORKSPACE}/config.xml\\\" // otherwise 'prolog not allowed in context'\\n xml = new XmlParser().parseText(jobconfig)\\n jenkinsfilepath = xml.children()['scriptPath'].text()\\n getYamlRoot = new File(jenkinsfilepath).getParent()\\n readYaml file: \\\"${getYamlRoot ?: env.WORKSPACE}\\\" + '/pipeline.yaml'\\n}\\n
\\nbut with Jenkins 2.401.3.3 the curl command gives
\\n\\n\\n- \\n
TLSv1.3 (IN), TLS handshake, Newsession Ticket (4):
\\n \\n- \\n
Mark bundle as not supporting multiuse
\\n< HTTP/1.1 403 Forbidden
\\n< Date: Tue, 29 Aug 2023 15:19:51 GMT
\\n< X-Content-Type-Options: nosniff
\\n< X-You-Are-Authenticated-As: xxx
\\n< X-You-Are-In-Group-Disabled: JENKINS-39402: use -Dhudson.security.AccessDeniedException2.REPORT_GROUP_HEADERS=true or use /whoAmI to diagnose
\\n< X-Required-Permission: hudson.model.Hudson.Read
\\n< X-Permission-Implied-By: hudson.security.Permission.GenericRead
\\n< X-Permission-Implied-By: hudson.model.Hudson.Administer
\\n< Content-Type: text/html;charset=utf-8
\\n< Expires: Thu, 01 Jan 1970 00:00:00 GMT
\\n< Cache-Control: no-cache,no-store,must-revalidate
\\n< X-Hudson-Theme: default
\\n< Referrer-Policy: same-origin
\\n< Cross-Origin-Opener-Policy: same-origin
\\n< Set-Cookie: JSESSIONID.7b2b95ba=node01m0vlt3zqe4oz11xw57rl6gglh96.node0; Path=/; Secure; HttpOnly
\\n< X-Hudson: 1.395
\\n< X-Jenkins: 2.401.3.3
\\n< X-Jenkins-Session: b904a4f9
\\n< X-Frame-Options: ALLOW-FROM http://xxx:81
\\n< X-Instance-Identity: xxx
\\n< Content-Length: 23013
\\n< Server: Jetty(10.0.13)
\\n<
\\n<head resURL=\\\"/static/b904a4f9\\\" data-rooturl=\\\"\\\" data-resurl=\\\"/static/b904a4f9\\\" [...]\\n \\n
\\n
\\nIn browser the response is the same for JOB_URL/config.xml in both Jenkins versions.
\\nThat\\u2019s also what i get as response from curl with Jenkins 2.375.4.2
\\n<flow-definition plugin=\\\"workflow-job@1344.cb-v917b_4f08a_b_5c\\\">\\n <actions>\\n <org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction plugin=\\\"workflow-multibranch@756.v891d88f2cd46\\\">\\n <jobPropertyDescriptors>\\n <string>jenkins.model.BuildDiscarderProperty</string>\\n </jobPropertyDescriptors>\\n </org.jenkinsci.plugins.workflow.multibranch.JobPropertyTrackerAction>\\n </actions>\\n <keepDependencies>false</keepDependencies>\\n <properties>\\n <org.jenkinsci.plugins.workflow.multibranch.BranchJobProperty plugin=\\\"workflow-multibranch@756.v891d88f2cd46\\\">\\n [...]\\n
\\nQuestions
\\nWhat\\u2019s the reason for the new behaviour ? Guess it\\u2019s some security related new feature, checked the change logs but didn\\u2019t find something helpful.
\\nIs there maybe a better / another way to get the Jenkinsfile path ? Searched a lot, but it seems there\\u2019s no other way than $JOB_URL/config.xml
\\nGilbert
\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"answers\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 341,\n \"samples\": [\n \"\\nThat\\u2019s what I need!
\\nThanks to this I found the solution:
\\n triggers {\\n BitbucketWebhookMultibranchTrigger {\\n refTrigger(true)\\n pullRequestTrigger(false)\\n }\\n }\\n
\\n@halkeys big thanks!
\",\n \"The problem probably your PATH setting via ENV ( it will not do what you expect\\u2026)
\\nI would create a (sh)-script for setting up the environment and call it from the jenkins job.
\",\n \"I\\u2019m not aware of changes here. More likely that you have an issue with permissions or your password/credential.
\\nYou should be able to get the scriptpath with groovy code like this (not tested). Though this will work only when you\\u2019re not in a sandbox. But within a shared library it works I guess.
\\n@NonCPS\\ndef getScriptPath() {\\n job = build.getParent()\\n definition = job.getDefinition()\\n if (definition instanceof org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition) {\\n return definition.scriptPath\\n }\\n return null\\n}\\n
\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}","type":"dataframe","variable_name":"df"},"text/html":["\n"," \n"," \n","\n","\n"," \n"," \n"," \n"," questions \n"," answers \n"," \n"," \n"," \n"," \n"," 0 \n"," <p>I use it to run autotests using python (pyt... \n"," <p>I would assume that the 2 GiB for the contr... \n"," \n"," \n","
\n","\n"," \n"," \n"],"text/plain":[" questions \\\n","0 I use it to run autotests using python (pyt... \n","\n"," answers \n","0
I would assume that the 2 GiB for the contr... "]},"execution_count":9,"metadata":{},"output_type":"execute_result"}],"source":["file_path = '../datasets/raw/QueryResults.csv'\n","\n","df = pd.read_csv(file_path)\n","\n","df.head(1)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"c4hpJk1MQLVb"},"outputs":[],"source":["df_updated = df.copy()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"SpYY2AIBKCac"},"outputs":[],"source":["df_updated['Question Body'] = df_updated['Question Body'].apply(remove_html_tags)\n","df_updated['Answer Body'] = df_updated['Answer Body'].apply(remove_html_tags)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Zn1jzfkLQFgP"},"outputs":[],"source":["# idx = 7\n","\n","# print(df['Question Body'][idx], df_updated['Question Body'][idx])"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"MntB0fEJKi5s"},"outputs":[],"source":["df_updated.to_csv('../datasets/QueryResultsUpdated.csv', index=False)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"VIAsqdZjc5Do"},"outputs":[],"source":[]}],"metadata":{"colab":{"provenance":[{"file_id":"1DL4jU9yDKJzg0BT8q9KDcz_5abtxe_4P","timestamp":1716256798358}]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}
2 |
--------------------------------------------------------------------------------
/FE/public/Jenkins_logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
--------------------------------------------------------------------------------
/src/data preprocessing/preprocessing.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Controllers"
8 | ]
9 | },
10 | {
11 | "cell_type": "code",
12 | "execution_count": 91,
13 | "metadata": {},
14 | "outputs": [],
15 | "source": [
16 | "file_name = '../../datasets/Jenkins Docs QA.csv'\n",
17 | "questions_col = 'Question'\n",
18 | "answers_col = 'Answer'\n",
19 | "\n",
20 | "# file_name = '../../datasets/Community Questions Refined.csv'\n",
21 | "# questions_col = 'questions'\n",
22 | "# answers_col = 'answers'\n",
23 | "\n",
24 | "# file_name = '../../datasets/QueryResultsUpdated.csv'\n",
25 | "# questions_col = 'Question Body'\n",
26 | "# answers_col = 'Answer Body'\n",
27 | "max_sequence_length = 11000\n",
28 | "num_samples = 1000 # None"
29 | ]
30 | },
31 | {
32 | "cell_type": "markdown",
33 | "metadata": {},
34 | "source": [
35 | "# Imports"
36 | ]
37 | },
38 | {
39 | "cell_type": "code",
40 | "execution_count": 92,
41 | "metadata": {},
42 | "outputs": [],
43 | "source": [
44 | "from utils import *\n",
45 | "import pandas as pd\n",
46 | "from datasets import Dataset"
47 | ]
48 | },
49 | {
50 | "cell_type": "markdown",
51 | "metadata": {},
52 | "source": [
53 | "# Load And Process Our Dataset - External Sources"
54 | ]
55 | },
56 | {
57 | "cell_type": "code",
58 | "execution_count": 93,
59 | "metadata": {},
60 | "outputs": [],
61 | "source": [
62 | "raw_data = pd.read_csv(file_name)"
63 | ]
64 | },
65 | {
66 | "cell_type": "code",
67 | "execution_count": 94,
68 | "metadata": {},
69 | "outputs": [
70 | {
71 | "data": {
72 | "text/plain": [
73 | "765"
74 | ]
75 | },
76 | "execution_count": 94,
77 | "metadata": {},
78 | "output_type": "execute_result"
79 | }
80 | ],
81 | "source": [
82 | "len(raw_data)"
83 | ]
84 | },
85 | {
86 | "cell_type": "code",
87 | "execution_count": 95,
88 | "metadata": {},
89 | "outputs": [
90 | {
91 | "data": {
92 | "text/html": [
93 | "
\n",
94 | "\n",
107 | "\n",
108 | " \n",
109 | " \n",
110 | " \n",
111 | " Question \n",
112 | " Answer \n",
113 | " \n",
114 | " \n",
115 | " \n",
116 | " \n",
117 | " 0 \n",
118 | " What is Jenkins? \n",
119 | " Jenkins is a self-contained, open-source autom... \n",
120 | " \n",
121 | " \n",
122 | " 1 \n",
123 | " What should I refer to if I want to develop my... \n",
124 | " If you want to extend the functionality of Jen... \n",
125 | " \n",
126 | " \n",
127 | "
\n",
128 | ""
129 | ],
130 | "text/plain": [
131 | " Question \n",
132 | "0 What is Jenkins? \\\n",
133 | "1 What should I refer to if I want to develop my... \n",
134 | "\n",
135 | " Answer \n",
136 | "0 Jenkins is a self-contained, open-source autom... \n",
137 | "1 If you want to extend the functionality of Jen... "
138 | ]
139 | },
140 | "execution_count": 95,
141 | "metadata": {},
142 | "output_type": "execute_result"
143 | }
144 | ],
145 | "source": [
146 | "raw_data.head(2)"
147 | ]
148 | },
149 | {
150 | "cell_type": "code",
151 | "execution_count": 96,
152 | "metadata": {},
153 | "outputs": [],
154 | "source": [
155 | "# Define condition to drop rows where answers_col has a code tag in\n",
156 | "condition = raw_data[answers_col].str.contains('')\n",
157 | "\n",
158 | "# Drop rows based on the condition\n",
159 | "raw_data = raw_data[~condition]"
160 | ]
161 | },
162 | {
163 | "cell_type": "code",
164 | "execution_count": 97,
165 | "metadata": {},
166 | "outputs": [
167 | {
168 | "data": {
169 | "text/plain": [
170 | "765"
171 | ]
172 | },
173 | "execution_count": 97,
174 | "metadata": {},
175 | "output_type": "execute_result"
176 | }
177 | ],
178 | "source": [
179 | "len(raw_data)"
180 | ]
181 | },
182 | {
183 | "cell_type": "code",
184 | "execution_count": 98,
185 | "metadata": {},
186 | "outputs": [
187 | {
188 | "data": {
189 | "text/html": [
190 | "\n",
191 | "\n",
204 | "\n",
205 | " \n",
206 | " \n",
207 | " \n",
208 | " Question \n",
209 | " Answer \n",
210 | " \n",
211 | " \n",
212 | " \n",
213 | " \n",
214 | " 0 \n",
215 | " What is Jenkins? \n",
216 | " Jenkins is a self-contained, open-source autom... \n",
217 | " \n",
218 | " \n",
219 | " 1 \n",
220 | " What should I refer to if I want to develop my... \n",
221 | " If you want to extend the functionality of Jen... \n",
222 | " \n",
223 | " \n",
224 | " 2 \n",
225 | " What is the purpose of this Jenkins documentat... \n",
226 | " The purpose of this documentation is to help u... \n",
227 | " \n",
228 | " \n",
229 | " 3 \n",
230 | " What is a good starting point for users who ha... \n",
231 | " Users who have never used Jenkins or have limi... \n",
232 | " \n",
233 | " \n",
234 | " 4 \n",
235 | " Where can I find more detailed information abo... \n",
236 | " For more detailed information about using Jenk... \n",
237 | " \n",
238 | " \n",
239 | "
\n",
240 | ""
241 | ],
242 | "text/plain": [
243 | " Question \n",
244 | "0 What is Jenkins? \\\n",
245 | "1 What should I refer to if I want to develop my... \n",
246 | "2 What is the purpose of this Jenkins documentat... \n",
247 | "3 What is a good starting point for users who ha... \n",
248 | "4 Where can I find more detailed information abo... \n",
249 | "\n",
250 | " Answer \n",
251 | "0 Jenkins is a self-contained, open-source autom... \n",
252 | "1 If you want to extend the functionality of Jen... \n",
253 | "2 The purpose of this documentation is to help u... \n",
254 | "3 Users who have never used Jenkins or have limi... \n",
255 | "4 For more detailed information about using Jenk... "
256 | ]
257 | },
258 | "execution_count": 98,
259 | "metadata": {},
260 | "output_type": "execute_result"
261 | }
262 | ],
263 | "source": [
264 | "raw_data.head(5)"
265 | ]
266 | },
267 | {
268 | "cell_type": "code",
269 | "execution_count": 99,
270 | "metadata": {},
271 | "outputs": [
272 | {
273 | "data": {
274 | "text/plain": [
275 | "0 Jenkins is a self-contained, open-source autom...\n",
276 | "1 If you want to extend the functionality of Jen...\n",
277 | "2 The purpose of this documentation is to help u...\n",
278 | "3 Users who have never used Jenkins or have limi...\n",
279 | "4 For more detailed information about using Jenk...\n",
280 | " ... \n",
281 | "760 File fingerprinting in Jenkins is used to trac...\n",
282 | "761 To set up file fingerprinting in Jenkins, rele...\n",
283 | "762 Jenkins stores the MD5 checksum (fingerprint) ...\n",
284 | "763 Jenkins users can benefit from file fingerprin...\n",
285 | "764 Some typical scenarios where file fingerprinti...\n",
286 | "Name: Answer, Length: 765, dtype: object"
287 | ]
288 | },
289 | "execution_count": 99,
290 | "metadata": {},
291 | "output_type": "execute_result"
292 | }
293 | ],
294 | "source": [
295 | "raw_data[answers_col]"
296 | ]
297 | },
298 | {
299 | "cell_type": "code",
300 | "execution_count": 100,
301 | "metadata": {},
302 | "outputs": [],
303 | "source": [
304 | "# Define condition to drop rows where questions_col has code tag\n",
305 | "condition = raw_data[questions_col].str.contains('')\n",
306 | "\n",
307 | "# Drop rows based on the condition\n",
308 | "raw_data = raw_data[~condition]"
309 | ]
310 | },
311 | {
312 | "cell_type": "code",
313 | "execution_count": 101,
314 | "metadata": {},
315 | "outputs": [
316 | {
317 | "data": {
318 | "text/plain": [
319 | "765"
320 | ]
321 | },
322 | "execution_count": 101,
323 | "metadata": {},
324 | "output_type": "execute_result"
325 | }
326 | ],
327 | "source": [
328 | "len(raw_data)"
329 | ]
330 | },
331 | {
332 | "cell_type": "code",
333 | "execution_count": 102,
334 | "metadata": {},
335 | "outputs": [
336 | {
337 | "data": {
338 | "text/html": [
339 | "\n",
340 | "\n",
353 | "\n",
354 | " \n",
355 | " \n",
356 | " \n",
357 | " Question \n",
358 | " Answer \n",
359 | " \n",
360 | " \n",
361 | " \n",
362 | " \n",
363 | " 0 \n",
364 | " What is Jenkins? \n",
365 | " Jenkins is a self-contained, open-source autom... \n",
366 | " \n",
367 | " \n",
368 | " 1 \n",
369 | " What should I refer to if I want to develop my... \n",
370 | " If you want to extend the functionality of Jen... \n",
371 | " \n",
372 | " \n",
373 | " 2 \n",
374 | " What is the purpose of this Jenkins documentat... \n",
375 | " The purpose of this documentation is to help u... \n",
376 | " \n",
377 | " \n",
378 | " 3 \n",
379 | " What is a good starting point for users who ha... \n",
380 | " Users who have never used Jenkins or have limi... \n",
381 | " \n",
382 | " \n",
383 | " 4 \n",
384 | " Where can I find more detailed information abo... \n",
385 | " For more detailed information about using Jenk... \n",
386 | " \n",
387 | " \n",
388 | "
\n",
389 | ""
390 | ],
391 | "text/plain": [
392 | " Question \n",
393 | "0 What is Jenkins? \\\n",
394 | "1 What should I refer to if I want to develop my... \n",
395 | "2 What is the purpose of this Jenkins documentat... \n",
396 | "3 What is a good starting point for users who ha... \n",
397 | "4 Where can I find more detailed information abo... \n",
398 | "\n",
399 | " Answer \n",
400 | "0 Jenkins is a self-contained, open-source autom... \n",
401 | "1 If you want to extend the functionality of Jen... \n",
402 | "2 The purpose of this documentation is to help u... \n",
403 | "3 Users who have never used Jenkins or have limi... \n",
404 | "4 For more detailed information about using Jenk... "
405 | ]
406 | },
407 | "execution_count": 102,
408 | "metadata": {},
409 | "output_type": "execute_result"
410 | }
411 | ],
412 | "source": [
413 | "raw_data.head()"
414 | ]
415 | },
416 | {
417 | "cell_type": "code",
418 | "execution_count": 104,
419 | "metadata": {},
420 | "outputs": [
421 | {
422 | "name": "stdout",
423 | "output_type": "stream",
424 | "text": [
425 | "length of our dataset: 765\n"
426 | ]
427 | }
428 | ],
429 | "source": [
430 | "# Combine columns into the desired format\n",
431 | "df = pd.DataFrame(columns=['text'])\n",
432 | "\n",
433 | "df['text'] = raw_data.apply(\n",
434 | " lambda row: f\"[INST] {row[questions_col].strip()} [/INST] {row[answers_col]} \",\n",
435 | " axis=1\n",
436 | ")\n",
437 | "print(f'length of our dataset: {len(df)}')"
438 | ]
439 | },
440 | {
441 | "cell_type": "code",
442 | "execution_count": 5,
443 | "metadata": {},
444 | "outputs": [
445 | {
446 | "name": "stdout",
447 | "output_type": "stream",
448 | "text": [
449 | "The maximum row length is: 30508\n"
450 | ]
451 | }
452 | ],
453 | "source": [
454 | "# Calculate the length of each row\n",
455 | "df['row_length'] = df['text'].apply(len)\n",
456 | "\n",
457 | "# Find the maximum row length\n",
458 | "max_length = df['row_length'].max()\n",
459 | "print(f\"The maximum row length is: {max_length}\")"
460 | ]
461 | },
462 | {
463 | "cell_type": "code",
464 | "execution_count": 6,
465 | "metadata": {},
466 | "outputs": [
467 | {
468 | "data": {
469 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHHCAYAAABeLEexAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABH+klEQVR4nO3deVwW5f7/8fctcCMugIhsqUhi7lrSyfi6YZKo2Kadk+WeZXWwcknN6mRZHctc2/ScU4mVbXZsOe7krmElSe6mZlIpYi4gqIhw/f7owfy6RQ0QuNF5PR+PeTycua575jNXN/Bu5pr7dhhjjAAAAGysirsLAAAAcDcCEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEeAGzzzzjBwOR4UcKyYmRjExMdb6qlWr5HA49Mknn1TI8QcNGqQGDRpUyLFKKzs7W/fdd59CQkLkcDg0fPhwd5dkO4mJiXI4HNq4caO7S4FNEYiAS1T4i7xwqVq1qsLCwhQXF6dXXnlFJ06cKJPjHDhwQM8884xSU1PLZH9lqTLXVhz//Oc/lZiYqIceekjvvvuu+vfvf8G+DRo0cPnvXb16dd1www165513KrDiC3M4HBo2bJi7y7igN954Q4mJie4uAyjC090FAFeKCRMmKCIiQnl5eUpPT9eqVas0fPhwTZ06VV988YVatWpl9X3qqaf0+OOPl2j/Bw4c0LPPPqsGDRro2muvLfbrli1bVqLjlMbFavvPf/6jgoKCcq/hUqxYsUI33nijxo8fX6z+1157rUaNGiVJOnjwoN58800NHDhQubm5uv/++8uz1MveG2+8ocDAQA0aNMjdpQAuCERAGenevbuuv/56a33cuHFasWKFevbsqVtvvVU7duyQj4+PJMnT01OenuX743fy5ElVq1ZNTqezXI/zZ7y8vNx6/OLIyMhQs2bNit3/qquuUr9+/az1QYMG6eqrr9a0adMIRMBliltmQDm66aab9I9//EP79+/Xe++9Z20/3xyipKQktW/fXv7+/qpRo4YaN26sJ554QtLv837+8pe/SJIGDx5s3a4pvPUQExOjFi1aKCUlRR07dlS1atWs1547h6hQfn6+nnjiCYWEhKh69eq69dZb9fPPP7v0adCgwXn/T/6P+/yz2s43hygnJ0ejRo1SvXr15O3trcaNG2vy5Mkyxrj0K7z989lnn6lFixby9vZW8+bNtWTJkvMP+DkyMjI0ZMgQBQcHq2rVqmrdurXmzJljtRfOp9q3b58WLlxo1f7TTz8Va/+F6tSpoyZNmmjv3r0lPs9evXqpTZs2Lq+75ZZb5HA49MUXX1jbvv76azkcDi1evLhEtZ1PQUGBpk+frubNm6tq1aoKDg7WAw88oGPHjrn0a9CggXr27Kl169bphhtuUNWqVXX11Vef9/bg5s2b1alTJ/n4+Khu3bp6/vnnNXv2bJfxbNCggbZt26bVq1dbY33uezM3N1cjR45UnTp1VL16dd1xxx06fPiwS5+NGzcqLi5OgYGB8vHxUUREhO69995LHhfYG1eIgHLWv39/PfHEE1q2bNkFrx5s27ZNPXv2VKtWrTRhwgR5e3trz549Wr9+vSSpadOmmjBhgp5++mkNHTpUHTp0kCT93//9n7WPI0eOqHv37urTp4/69eun4ODgi9b1wgsvyOFwaOzYscrIyND06dMVGxur1NRU60pWcRSntj8yxujWW2/VypUrNWTIEF177bVaunSpRo8erV9//VXTpk1z6b9u3TrNnz9ff//731WzZk298sor6t27t9LS0lS7du0L1nXq1CnFxMRoz549GjZsmCIiIjRv3jwNGjRIx48f16OPPqqmTZvq3Xff1YgRI1S3bl3rNlidOnWKff6SdPbsWf3yyy+qVatWic+zQ4cO+vzzz5WVlSVfX18ZY7R+/XpVqVJFa9eu1a233ipJWrt2rapUqaJ27dqVqLbzeeCBB5SYmKjBgwfrkUce0b59+/Taa69p06ZNWr9+vctVvT179ujOO+/UkCFDNHDgQL399tsaNGiQoqKi1Lx5c0nSr7/+qs6dO8vhcGjcuHGqXr263nzzTXl7e7scd/r06Xr44YdVo0YNPfnkk5JU5H368MMPq1atWho/frx++uknTZ8+XcOGDdNHH30k6feQ27VrV9WpU0ePP/64/P399dNPP2n+/PmXPC6wOQPgksyePdtIMt9+++0F+/j5+ZnrrrvOWh8/frz544/ftGnTjCRz+PDhC+7j22+/NZLM7Nmzi7R16tTJSDKzZs06b1unTp2s9ZUrVxpJ5qqrrjJZWVnW9o8//thIMjNmzLC2hYeHm4EDB/7pPi9W28CBA014eLi1/tlnnxlJ5vnnn3fpd+eddxqHw2H27NljbZNknE6ny7bvv//eSDKvvvpqkWP90fTp040k895771nbzpw5Y6Kjo02NGjVczj08PNzEx8dfdH9/7Nu1a1dz+PBhc/jwYbNlyxbTv39/I8kkJCSU+DwLx27RokXGGGM2b95sJJm//vWvpm3bttbrbr31Vpf30IWcW8e51q5daySZuXPnumxfsmRJke3h4eFGklmzZo21LSMjw3h7e5tRo0ZZ2x5++GHjcDjMpk2brG1HjhwxAQEBRpLZt2+ftb158+Yu751ChT9HsbGxpqCgwNo+YsQI4+HhYY4fP26MMebTTz/90583oDS4ZQZUgBo1alz0aTN/f39J0ueff17qCcje3t4aPHhwsfsPGDBANWvWtNbvvPNOhYaGatGiRaU6fnEtWrRIHh4eeuSRR1y2jxo1SsaYIreEYmNj1bBhQ2u9VatW8vX11Y8//vinxwkJCdHdd99tbfPy8tIjjzyi7OxsrV69utTnsGzZMtWpU0d16tRRy5Yt9e6772rw4MF6+eWXS3ye1113nWrUqKE1a9ZI+v1KUN26dTVgwAB99913OnnypIwxWrdunXX17VLMmzdPfn5+uvnmm/Xbb79ZS1RUlGrUqKGVK1e69G/WrJnLcevUqaPGjRu7jP+SJUsUHR3tMqE+ICBAffv2LXF9Q4cOdbmd3KFDB+Xn52v//v2S/v/PyoIFC5SXl1fi/QMXQiACKkB2drZL+DjXXXfdpXbt2um+++5TcHCw+vTpo48//rhE4eiqq64q0QTqRo0auaw7HA5FRkaWeP5MSe3fv19hYWFFxqNp06ZW+x/Vr1+/yD5q1apVZL7L+Y7TqFEjVani+mvuQscpibZt2yopKUlLlizR5MmT5e/vr2PHjrmMf3HP08PDQ9HR0Vq7dq2k3wNRhw4d1L59e+Xn52vDhg3avn27jh49WiaBaPfu3crMzFRQUJAV6gqX7OxsZWRkuPQvzvjv379fkZGRRfqdb9ufOfd4hbchC4/XqVMn9e7dW88++6wCAwN12223afbs2crNzS3xsYA/Yg4RUM5++eUXZWZmXvSPg4+Pj9asWaOVK1dq4cKFWrJkiT766CPddNNNWrZsmTw8PP70OCWZ91NcF/rwyPz8/GLVVBYudBxzzgTsihQYGKjY2FhJUlxcnJo0aaKePXtqxowZGjlyZIn31759e73wwgs6ffq01q5dqyeffFL+/v5q0aKF1q5da82zKYtAVFBQoKCgIM2dO/e87efOn6ro8f+z4xV+qOiGDRv0v//9T0uXLtW9996rKVOmaMOGDapRo0a51IUrH1eIgHL27rvvSvr9D+fFVKlSRV26dNHUqVO1fft2vfDCC1qxYoV1C6OsP9l69+7dLuvGGO3Zs8flibBatWrp+PHjRV577tWVktQWHh6uAwcOFLmFuHPnTqu9LISHh2v37t1FrrKV9XEkKT4+Xp06ddI///lP5eTkWPsv7nl26NBBZ86c0QcffKBff/3VCj4dO3bU2rVrtXbtWl1zzTV/OlG+OBo2bKgjR46oXbt2io2NLbK0bt26xPsMDw/Xnj17imw/37ayeh/feOONeuGFF7Rx40bNnTtX27Zt04cfflgm+4Y9EYiAcrRixQo999xzioiIuOh8iqNHjxbZVjgfo/BWQPXq1SXpvAGlNN555x2XP9affPKJDh48qO7du1vbGjZsqA0bNujMmTPWtgULFhR5PL8ktfXo0UP5+fl67bXXXLZPmzZNDofD5fiXokePHkpPT7eeTpJ+fxrs1VdfVY0aNdSpU6cyOU6hsWPH6siRI/rPf/5jHb+459m2bVt5eXnppZdeUkBAgPX0VocOHbRhwwatXr26TK4OSdLf/vY35efn67nnnivSdvbs2VK9v+Li4pScnOzySeVHjx4971Wo6tWrX9J7+NixY0WuTp37swKUBrfMgDKyePFi7dy5U2fPntWhQ4e0YsUKJSUlKTw8XF988YWqVq16wddOmDBBa9asUXx8vMLDw5WRkaE33nhDdevWVfv27SX9Hk78/f01a9Ys1axZU9WrV1fbtm0VERFRqnoDAgLUvn17DR48WIcOHdL06dMVGRnp8tEA9913nz755BN169ZNf/vb37R371699957LpOcS1rbLbfcos6dO+vJJ5/UTz/9pNatW2vZsmX6/PPPNXz48CL7Lq2hQ4fqX//6lwYNGqSUlBQ1aNBAn3zyidavX6/p06dfdE5XaXTv3l0tWrTQ1KlTlZCQUKLzrFatmqKiorRhwwbrM4ik368Q5eTkKCcnp0SBaOPGjXr++eeLbI+JiVGnTp30wAMPaOLEiUpNTVXXrl3l5eWl3bt3a968eZoxY4buvPPOEp37mDFj9N577+nmm2/Www8/bD12X79+fR09etTlqlBUVJRmzpyp559/XpGRkQoKCtJNN91U7GPNmTNHb7zxhu644w41bNhQJ06c0H/+8x/5+vqqR48eJaobcOG+B9yAK0Ph48KFi9PpNCEhIebmm282M2bMcHm8u9C5j90vX77c3HbbbSYsLMw4nU4TFhZm7r77bvPDDz+4vO7zzz83zZo1M56eni6PuXfq1Mk0b978vPVd6LH7Dz74wIwbN84EBQUZHx8fEx8fb/bv31/k9VOmTDFXXXWV8fb2Nu3atTMbN24sss+L1XbuY/fGGHPixAkzYsQIExYWZry8vEyjRo3Myy+/7PK4tTEXfoT8Qh8HcK5Dhw6ZwYMHm8DAQON0Ok3Lli3P+9EAJX3s/kJ9ExMTXc69uOdpjDGjR482ksxLL73ksj0yMtJIMnv37i1WfX98L567PPfcc1a/f//73yYqKsr4+PiYmjVrmpYtW5oxY8aYAwcO/Om5nu+//6ZNm0yHDh2Mt7e3qVu3rpk4caJ55ZVXjCSTnp5u9UtPTzfx8fGmZs2aRpK1nwt9fEXh+3XlypXGGGO+++47c/fdd5v69esbb29vExQUZHr27Gk2btxYrPEBLsRhjBtnJgIArljDhw/Xv/71L2VnZ1fYJHygtJhDBAC4ZKdOnXJZP3LkiN599121b9+eMITLAnOIAACXLDo6WjExMWratKkOHTqkt956S1lZWfrHP/7h7tKAYiEQAQAuWY8ePfTJJ5/o3//+txwOh9q0aaO33npLHTt2dHdpQLEwhwgAANgec4gAAIDtEYgAAIDtMYeoGAoKCnTgwAHVrFmzzL8+AQAAlA9jjE6cOKGwsLAiX/R8LgJRMRw4cED16tVzdxkAAKAUfv75Z9WtW/eifQhExVD4Ef8///yzfH193VwNAAAojqysLNWrV69YX9VDICqGwttkvr6+BCIAAC4zxZnu4tZJ1TNnzlSrVq2soBEdHa3Fixdb7adPn1ZCQoJq166tGjVqqHfv3jp06JDLPtLS0hQfH69q1aopKChIo0eP1tmzZ136rFq1Sm3atJG3t7ciIyOVmJhYEacHAAAuE24NRHXr1tWLL76olJQUbdy4UTfddJNuu+02bdu2TZI0YsQI/e9//9O8efO0evVqHThwQL169bJen5+fr/j4eJ05c0ZfffWV5syZo8TERD399NNWn3379ik+Pl6dO3dWamqqhg8frvvuu09Lly6t8PMFAACVU6X7YMaAgAC9/PLLuvPOO1WnTh29//77uvPOOyVJO3fuVNOmTZWcnKwbb7xRixcvVs+ePXXgwAEFBwdLkmbNmqWxY8fq8OHDcjqdGjt2rBYuXKitW7dax+jTp4+OHz+uJUuWFKumrKws+fn5KTMzk1tmAABcJkry97vSfA5Rfn6+PvzwQ+Xk5Cg6OlopKSnKy8tTbGys1adJkyaqX7++kpOTJUnJyclq2bKlFYYkKS4uTllZWdZVpuTkZJd9FPYp3AcAAIDbJ1Vv2bJF0dHROn36tGrUqKFPP/1UzZo1U2pqqpxOp/z9/V36BwcHKz09XZKUnp7uEoYK2wvbLtYnKytLp06dko+PT5GacnNzlZuba61nZWVd8nkCAIDKy+1XiBo3bqzU1FR9/fXXeuihhzRw4EBt377drTVNnDhRfn5+1sJnEAEAcGVzeyByOp2KjIxUVFSUJk6cqNatW2vGjBkKCQnRmTNndPz4cZf+hw4dUkhIiCQpJCSkyFNnhet/1sfX1/e8V4ckady4ccrMzLSWn3/+uSxOFQAAVFJuD0TnKigoUG5urqKiouTl5aXly5dbbbt27VJaWpqio6MlSdHR0dqyZYsyMjKsPklJSfL19VWzZs2sPn/cR2Gfwn2cj7e3t/VRAHz2EAAAVz63ziEaN26cunfvrvr16+vEiRN6//33tWrVKi1dulR+fn4aMmSIRo4cqYCAAPn6+urhhx9WdHS0brzxRklS165d1axZM/Xv31+TJk1Senq6nnrqKSUkJMjb21uS9OCDD+q1117TmDFjdO+992rFihX6+OOPtXDhQneeOgAAqETcGogyMjI0YMAAHTx4UH5+fmrVqpWWLl2qm2++WZI0bdo0ValSRb1791Zubq7i4uL0xhtvWK/38PDQggUL9NBDDyk6OlrVq1fXwIEDNWHCBKtPRESEFi5cqBEjRmjGjBmqW7eu3nzzTcXFxVX4+QIAgMqp0n0OUWXE5xABAHD5uSw/hwgAAMBdCEQAAMD2CEQAAMD23P5J1ZDS0tL022+/ubuMEgkMDFT9+vXdXQYAAGWCQORmaWlpatykqU6fOunuUkqkqk817dq5g1AEALgiEIjc7LffftPpUydVu+coedW+PL4iJO/IzzqyYIp+++03AhEA4IpAIKokvGrXk3dIpLvLAADAlphUDQAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbI9ABAAAbM+tgWjixIn6y1/+opo1ayooKEi33367du3a5dInJiZGDofDZXnwwQdd+qSlpSk+Pl7VqlVTUFCQRo8erbNnz7r0WbVqldq0aSNvb29FRkYqMTGxvE8PAABcJtwaiFavXq2EhARt2LBBSUlJysvLU9euXZWTk+PS7/7779fBgwetZdKkSVZbfn6+4uPjdebMGX311VeaM2eOEhMT9fTTT1t99u3bp/j4eHXu3FmpqakaPny47rvvPi1durTCzhUAAFRenu48+JIlS1zWExMTFRQUpJSUFHXs2NHaXq1aNYWEhJx3H8uWLdP27dv15ZdfKjg4WNdee62ee+45jR07Vs8884ycTqdmzZqliIgITZkyRZLUtGlTrVu3TtOmTVNcXFz5nSAAALgsVKo5RJmZmZKkgIAAl+1z585VYGCgWrRooXHjxunkyZNWW3Jyslq2bKng4GBrW1xcnLKysrRt2zarT2xsrMs+4+LilJycfN46cnNzlZWV5bIAAIArl1uvEP1RQUGBhg8frnbt2qlFixbW9nvuuUfh4eEKCwvT5s2bNXbsWO3atUvz58+XJKWnp7uEIUnWenp6+kX7ZGVl6dSpU/Lx8XFpmzhxop599tkyP0cAAFA5VZpAlJCQoK1bt2rdunUu24cOHWr9u2XLlgoNDVWXLl20d+9eNWzYsFxqGTdunEaOHGmtZ2VlqV69euVyLAAA4H6V4pbZsGHDtGDBAq1cuVJ169a9aN+2bdtKkvbs2SNJCgkJ0aFDh1z6FK4Xzju6UB9fX98iV4ckydvbW76+vi4LAAC4crk1EBljNGzYMH366adasWKFIiIi/vQ1qampkqTQ0FBJUnR0tLZs2aKMjAyrT1JSknx9fdWsWTOrz/Lly132k5SUpOjo6DI6EwAAcDlzayBKSEjQe++9p/fff181a9ZUenq60tPTderUKUnS3r179dxzzyklJUU//fSTvvjiCw0YMEAdO3ZUq1atJEldu3ZVs2bN1L9/f33//fdaunSpnnrqKSUkJMjb21uS9OCDD+rHH3/UmDFjtHPnTr3xxhv6+OOPNWLECLedOwAAqDzcGohmzpypzMxMxcTEKDQ01Fo++ugjSZLT6dSXX36prl27qkmTJho1apR69+6t//3vf9Y+PDw8tGDBAnl4eCg6Olr9+vXTgAEDNGHCBKtPRESEFi5cqKSkJLVu3VpTpkzRm2++ySP3AABAkpsnVRtjLtper149rV69+k/3Ex4erkWLFl20T0xMjDZt2lSi+gAAgD1UiknVAAAA7kQgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtkcgAgAAtufWQDRx4kT95S9/Uc2aNRUUFKTbb79du3btculz+vRpJSQkqHbt2qpRo4Z69+6tQ4cOufRJS0tTfHy8qlWrpqCgII0ePVpnz5516bNq1Sq1adNG3t7eioyMVGJiYnmfHgAAuEy4NRCtXr1aCQkJ2rBhg5KSkpSXl6euXbsqJyfH6jNixAj973//07x587R69WodOHBAvXr1strz8/MVHx+vM2fO6KuvvtKcOXOUmJiop59+2uqzb98+xcfHq3PnzkpNTdXw4cN13333aenSpRV6vgAAoHJyGGOMu4sodPjwYQUFBWn16tXq2LGjMjMzVadOHb3//vu68847JUk7d+5U06ZNlZycrBtvvFGLFy9Wz549deDAAQUHB0uSZs2apbFjx+rw4cNyOp0aO3asFi5cqK1bt1rH6tOnj44fP64lS5b8aV1ZWVny8/NTZmamfH19y/Scv/vuO0VFRSlk4HR5h0SW6b7LS276HqXPGa6UlBS1adPG3eUAAHBeJfn7XanmEGVmZkqSAgICJEkpKSnKy8tTbGys1adJkyaqX7++kpOTJUnJyclq2bKlFYYkKS4uTllZWdq2bZvV54/7KOxTuI9z5ebmKisry2UBAABXrkoTiAoKCjR8+HC1a9dOLVq0kCSlp6fL6XTK39/fpW9wcLDS09OtPn8MQ4XthW0X65OVlaVTp04VqWXixIny8/Ozlnr16pXJOQIAgMqp0gSihIQEbd26VR9++KG7S9G4ceOUmZlpLT///LO7SwIAAOXI090FSNKwYcO0YMECrVmzRnXr1rW2h4SE6MyZMzp+/LjLVaJDhw4pJCTE6vPNN9+47K/wKbQ/9jn3ybRDhw7J19dXPj4+Rerx9vaWt7d3mZwbAACo/Nx6hcgYo2HDhunTTz/VihUrFBER4dIeFRUlLy8vLV++3Nq2a9cupaWlKTo6WpIUHR2tLVu2KCMjw+qTlJQkX19fNWvWzOrzx30U9incBwAAsDe3XiFKSEjQ+++/r88//1w1a9a05vz4+fnJx8dHfn5+GjJkiEaOHKmAgAD5+vrq4YcfVnR0tG688UZJUteuXdWsWTP1799fkyZNUnp6up566iklJCRYV3kefPBBvfbaaxozZozuvfderVixQh9//LEWLlzotnMHAACVh1uvEM2cOVOZmZmKiYlRaGiotXz00UdWn2nTpqlnz57q3bu3OnbsqJCQEM2fP99q9/Dw0IIFC+Th4aHo6Gj169dPAwYM0IQJE6w+ERERWrhwoZKSktS6dWtNmTJFb775puLi4ir0fAEAQOXk1itExfkIpKpVq+r111/X66+/fsE+4eHhWrRo0UX3ExMTo02bNpW4RgAAcOWrNE+ZAQAAuAuBCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2F6pAtGPP/5Y1nUAAAC4TakCUWRkpDp37qz33ntPp0+fLuuaAAAAKlSpAtF3332nVq1aaeTIkQoJCdEDDzygb775pqxrAwAAqBClCkTXXnutZsyYoQMHDujtt9/WwYMH1b59e7Vo0UJTp07V4cOHy7pOAACAcnNJk6o9PT3Vq1cvzZs3Ty+99JL27Nmjxx57TPXq1dOAAQN08ODBsqoTAACg3FxSINq4caP+/ve/KzQ0VFOnTtVjjz2mvXv3KikpSQcOHNBtt91WVnUCAACUG8/SvGjq1KmaPXu2du3apR49euidd95Rjx49VKXK7/kqIiJCiYmJatCgQVnWCgAAUC5KFYhmzpype++9V4MGDVJoaOh5+wQFBemtt966pOIAAAAqQqkC0e7du/+0j9Pp1MCBA0uzewAAgApVqjlEs2fP1rx584psnzdvnubMmXPJRQEAAFSkUgWiiRMnKjAwsMj2oKAg/fOf/7zkogAAACpSqQJRWlqaIiIiimwPDw9XWlraJRcFAABQkUoViIKCgrR58+Yi27///nvVrl37kosCAACoSKUKRHfffbceeeQRrVy5Uvn5+crPz9eKFSv06KOPqk+fPmVdIwAAQLkq1VNmzz33nH766Sd16dJFnp6/76KgoEADBgxgDhEAALjslCoQOZ1OffTRR3ruuef0/fffy8fHRy1btlR4eHhZ1wcAAFDuShWICl1zzTW65ppryqoWAAAAtyhVIMrPz1diYqKWL1+ujIwMFRQUuLSvWLGiTIoDAACoCKUKRI8++qgSExMVHx+vFi1ayOFwlHVdAAAAFaZUgejDDz/Uxx9/rB49epR1PQAAABWuVI/dO51ORUZGlnUtAAAAblGqQDRq1CjNmDFDxpiyrgcAAKDCleqW2bp167Ry5UotXrxYzZs3l5eXl0v7/Pnzy6Q4AACAilCqQOTv76877rijrGsBAABwi1IFotmzZ5d1HQAAAG5TqjlEknT27Fl9+eWX+te//qUTJ05Ikg4cOKDs7OwyKw4AAKAilOoK0f79+9WtWzelpaUpNzdXN998s2rWrKmXXnpJubm5mjVrVlnXCQAAUG5KdYXo0Ucf1fXXX69jx47Jx8fH2n7HHXdo+fLlZVYcAABARSjVFaK1a9fqq6++ktPpdNneoEED/frrr2VSGAAAQEUp1RWigoIC5efnF9n+yy+/qGbNmpdcFAAAQEUqVSDq2rWrpk+fbq07HA5lZ2dr/PjxfJ0HAAC47JTqltmUKVMUFxenZs2a6fTp07rnnnu0e/duBQYG6oMPPijrGgEAAMpVqa4Q1a1bV99//72eeOIJjRgxQtddd51efPFFbdq0SUFBQcXez5o1a3TLLbcoLCxMDodDn332mUv7oEGD5HA4XJZu3bq59Dl69Kj69u0rX19f+fv7a8iQIUUe/d+8ebM6dOigqlWrql69epo0aVJpThsAAFyhSnWFSJI8PT3Vr1+/Szp4Tk6OWrdurXvvvVe9evU6b59u3bq5fBCkt7e3S3vfvn118OBBJSUlKS8vT4MHD9bQoUP1/vvvS5KysrLUtWtXxcbGatasWdqyZYvuvfde+fv7a+jQoZdUPwAAuDKUKhC98847F20fMGBAsfbTvXt3de/e/aJ9vL29FRISct62HTt2aMmSJfr22291/fXXS5JeffVV9ejRQ5MnT1ZYWJjmzp2rM2fO6O2335bT6VTz5s2VmpqqqVOnEogAAICkUgaiRx991GU9Ly9PJ0+elNPpVLVq1YodiIpj1apVCgoKUq1atXTTTTfp+eefV+3atSVJycnJ8vf3t8KQJMXGxqpKlSr6+uuvdccddyg5OVkdO3Z0+YiAuLg4vfTSSzp27Jhq1apVZrUCAIDLU6kC0bFjx4ps2717tx566CGNHj36kosq1K1bN/Xq1UsRERHau3evnnjiCXXv3l3Jycny8PBQenp6kTlLnp6eCggIUHp6uiQpPT1dERERLn2Cg4OttvMFotzcXOXm5lrrWVlZZXZOAACg8in1HKJzNWrUSC+++KL69eunnTt3lsk++/TpY/27ZcuWatWqlRo2bKhVq1apS5cuZXKM85k4caKeffbZcts/AACoXEr95a7n4+npqQMHDpTlLl1cffXVCgwM1J49eyRJISEhysjIcOlz9uxZHT161Jp3FBISokOHDrn0KVy/0NykcePGKTMz01p+/vnnsj4VAABQiZTqCtEXX3zhsm6M0cGDB/Xaa6+pXbt2ZVLY+fzyyy86cuSIQkNDJUnR0dE6fvy4UlJSFBUVJUlasWKFCgoK1LZtW6vPk08+qby8PHl5eUmSkpKS1Lhx4wvOH/L29i7yNBsAALhylSoQ3X777S7rDodDderU0U033aQpU6YUez/Z2dnW1R5J2rdvn1JTUxUQEKCAgAA9++yz6t27t0JCQrR3716NGTNGkZGRiouLkyQ1bdpU3bp10/33369Zs2YpLy9Pw4YNU58+fRQWFiZJuueee/Tss89qyJAhGjt2rLZu3aoZM2Zo2rRppTl1AABwBSpVICooKCiTg2/cuFGdO3e21keOHClJGjhwoGbOnKnNmzdrzpw5On78uMLCwtS1a1c999xzLldv5s6dq2HDhqlLly6qUqWKevfurVdeecVq9/Pz07Jly5SQkKCoqCgFBgbq6aef5pF7AABgKbNJ1aURExMjY8wF25cuXfqn+wgICLA+hPFCWrVqpbVr15a4PgAAYA+lCkSFV3KKY+rUqaU5BAAAQIUpVSDatGmTNm3apLy8PDVu3FiS9MMPP8jDw0Nt2rSx+jkcjrKpEgAAoByVKhDdcsstqlmzpubMmWM9qXXs2DENHjxYHTp00KhRo8q0SAAAgPJUqs8hmjJliiZOnOjy2HqtWrX0/PPPl+gpMwAAgMqgVIEoKytLhw8fLrL98OHDOnHixCUXBQAAUJFKFYjuuOMODR48WPPnz9cvv/yiX375Rf/97381ZMgQ9erVq6xrBAAAKFelmkM0a9YsPfbYY7rnnnuUl5f3+448PTVkyBC9/PLLZVogAABAeStVIKpWrZreeOMNvfzyy9q7d68kqWHDhqpevXqZFgcAAFARLunLXQ8ePKiDBw+qUaNGql69+kU/ZBEAAKCyKlUgOnLkiLp06aJrrrlGPXr00MGDByVJQ4YM4ZF7AABw2SlVIBoxYoS8vLyUlpamatWqWdvvuusuLVmypMyKAwAAqAilmkO0bNkyLV26VHXr1nXZ3qhRI+3fv79MCgMAAKgopbpClJOT43JlqNDRo0ddvokeAADgclCqQNShQwe988471rrD4VBBQYEmTZqkzp07l1lxAAAAFaFUt8wmTZqkLl26aOPGjTpz5ozGjBmjbdu26ejRo1q/fn1Z1wgAAFCuSnWFqEWLFvrhhx/Uvn173XbbbcrJyVGvXr20adMmNWzYsKxrBAAAKFclvkKUl5enbt26adasWXryySfLoyYAAIAKVeIrRF5eXtq8eXN51AIAAOAWpbpl1q9fP7311ltlXQsAAIBblGpS9dmzZ/X222/ryy+/VFRUVJHvMJs6dWqZFAcAAFARShSIfvzxRzVo0EBbt25VmzZtJEk//PCDSx+Hw1F21QEAAFSAEgWiRo0a6eDBg1q5cqWk37+q45VXXlFwcHC5FAcAAFARSjSH6Nxvs1+8eLFycnLKtCAAAICKVqpJ1YXODUgAAACXoxIFIofDUWSOEHOGAADA5a5Ec4iMMRo0aJD1Ba6nT5/Wgw8+WOQps/nz55ddhQAAAOWsRIFo4MCBLuv9+vUr02IAAADcoUSBaPbs2eVVBwAAgNtc0qRqAACAKwGBCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2J5bA9GaNWt0yy23KCwsTA6HQ5999plLuzFGTz/9tEJDQ+Xj46PY2Fjt3r3bpc/Ro0fVt29f+fr6yt/fX0OGDFF2drZLn82bN6tDhw6qWrWq6tWrp0mTJpX3qQEAgMuIWwNRTk6OWrdurddff/287ZMmTdIrr7yiWbNm6euvv1b16tUVFxen06dPW3369u2rbdu2KSkpSQsWLNCaNWs0dOhQqz0rK0tdu3ZVeHi4UlJS9PLLL+uZZ57Rv//973I/PwAAcHnwdOfBu3fvru7du5+3zRij6dOn66mnntJtt90mSXrnnXcUHByszz77TH369NGOHTu0ZMkSffvtt7r++uslSa+++qp69OihyZMnKywsTHPnztWZM2f09ttvy+l0qnnz5kpNTdXUqVNdghMAALCvSjuHaN++fUpPT1dsbKy1zc/PT23btlVycrIkKTk5Wf7+/lYYkqTY2FhVqVJFX3/9tdWnY8eOcjqdVp+4uDjt2rVLx44dO++xc3NzlZWV5bIAAIArV6UNROnp6ZKk4OBgl+3BwcFWW3p6uoKCglzaPT09FRAQ4NLnfPv44zHONXHiRPn5+VlLvXr1Lv2EAABApVVpA5E7jRs3TpmZmdby888/u7skAABQjiptIAoJCZEkHTp0yGX7oUOHrLaQkBBlZGS4tJ89e1ZHjx516XO+ffzxGOfy9vaWr6+vywIAAK5clTYQRUREKCQkRMuXL7e2ZWVl6euvv1Z0dLQkKTo6WsePH1dKSorVZ8WKFSooKFDbtm2tPmvWrFFeXp7VJykpSY0bN1atWrUq6GwAAEBl5tZAlJ2drdTUVKWmpkr6fSJ1amqq0tLS5HA4NHz4cD3//PP64osvtGXLFg0YMEBhYWG6/fbbJUlNmzZVt27ddP/99+ubb77R+vXrNWzYMPXp00dhYWGSpHvuuUdOp1NDhgzRtm3b9NFHH2nGjBkaOXKkm84aAABUNm597H7jxo3q3LmztV4YUgYOHKjExESNGTNGOTk5Gjp0qI4fP6727dtryZIlqlq1qvWauXPnatiwYerSpYuqVKmi3r1765VXXrHa/fz8tGzZMiUkJCgqKkqBgYF6+umneeQeAABYHMYY4+4iKrusrCz5+fkpMzOzzOcTfffdd4qKilLIwOnyDoks032Xl9z0PUqfM1wpKSlq06aNu8sBAOC8SvL3u9LOIQIAAKgoBCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7lToQPfPMM3I4HC5LkyZNrPbTp08rISFBtWvXVo0aNdS7d28dOnTIZR9paWmKj49XtWrVFBQUpNGjR+vs2bMVfSoAAKAS83R3AX+mefPm+vLLL611T8//X/KIESO0cOFCzZs3T35+fho2bJh69eql9evXS5Ly8/MVHx+vkJAQffXVVzp48KAGDBggLy8v/fOf/6zwcwEAAJVTpQ9Enp6eCgkJKbI9MzNTb731lt5//33ddNNNkqTZs2eradOm2rBhg2688UYtW7ZM27dv15dffqng4GBde+21eu655zR27Fg988wzcjqdFX06AACgEqrUt8wkaffu3QoLC9PVV1+tvn37Ki0tTZKUkpKivLw8xcbGWn2bNGmi+vXrKzk5WZKUnJysli1bKjg42OoTFxenrKwsbdu27YLHzM3NVVZWlssCAACuXJU6ELVt21aJiYlasmSJZs6cqX379qlDhw46ceKE0tPT5XQ65e/v7/Ka4OBgpaenS5LS09NdwlBhe2HbhUycOFF+fn7WUq9evbI9MQAAUKlU6ltm3bt3t/7dqlUrtW3bVuHh4fr444/l4+NTbscdN26cRo4caa1nZWURigAAuIJV6itE5/L399c111yjPXv2KCQkRGfOnNHx48dd+hw6dMiacxQSElLkqbPC9fPNSyrk7e0tX19flwUAAFy5LqtAlJ2drb179yo0NFRRUVHy8vLS8uXLrfZdu3YpLS1N0dHRkqTo6Ght2bJFGRkZVp+kpCT5+vqqWbNmFV4/AAConCr1LbPHHntMt9xyi8LDw3XgwAGNHz9eHh4euvvuu+Xn56chQ4Zo5MiRCggIkK+vrx5++GFFR0frxhtvlCR17dpVzZo1U//+/TVp0iSlp6frqaeeUkJCgry9vd18dgAAoLKo1IHol19+0d13360jR46oTp06at++vTZs2KA6depIkqZNm6YqVaqod+/eys3NVVxcnN544w3r9R4eHlqwYIEeeughRUdHq3r16ho4cKAmTJjgrlMCAACVUKUORB9++OFF26tWrarXX39dr7/++gX7hIeHa9GiRWVdGgAAuIJcVnOIAAAAygOBCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2B6BCAAA2F6l/mBGVG47duxwdwklEhgYqPr167u7DABAJUQgQonlZx+THA7169fP3aWUSFWfatq1cwehCABQBIEIJVaQmy0Zo9o9R8mrdj13l1MseUd+1pEFU/Tbb78RiAAARRCIUGpetevJOyTS3WUAAHDJmFQNAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsj0AEAABsz9PdBQAVaceOHe4uoUQCAwNVv359d5cBAFc8AhFsIT/7mORwqF+/fu4upUSq+lTTrp07CEUAUM4IRLCFgtxsyRjV7jlKXrXrubucYsk78rOOLJii3377jUAEAOWMQARb8apdT94hke4uAwBQyTCpGgAA2B6BCAAA2B6BCAAA2B6BCAAA2J6tAtHrr7+uBg0aqGrVqmrbtq2++eYbd5cEAAAqAdsEoo8++kgjR47U+PHj9d1336l169aKi4tTRkaGu0sDAABuZpvH7qdOnar7779fgwcPliTNmjVLCxcu1Ntvv63HH3/czdUBF8anawNA+bNFIDpz5oxSUlI0btw4a1uVKlUUGxur5ORkN1YGXNjl+una3t5V9d//fqLQ0FB3l1Jsubm58vb2dncZJXI51ixdnnUT8u3BFoHot99+U35+voKDg122BwcHa+fOnUX65+bmKjc311rPzMyUJGVlZZV5bdnZ2b8fM32PCs6cLvP9l4e8Iz9Loubylntgh2SMfP/SSx5+ddxdTrHkHf5J2d8vVc+ePd1dSgk5JBl3F1FCl2PN0uVYt9O7qt57950if0MqsypVqqigoMDdZZRISEiIQkJCynSfhX+3jfnz95wtAlFJTZw4Uc8++2yR7fXqld9XPhxb+lq57bu8UHPFyPp2vrtLsIHL6w/07y7HmqXLse4zuaf1t7/9zd1l4BKcOHFCfn5+F+1ji0AUGBgoDw8PHTp0yGX7oUOHzptGx40bp5EjR1rrBQUFOnr0qGrXri2Hw3HJ9WRlZalevXr6+eef5evre8n7u5IxVsXDOBUP41R8jFXxME7F465xMsboxIkTCgsL+9O+tghETqdTUVFRWr58uW6//XZJv4ec5cuXa9iwYUX6e3t7F7nH7e/vX+Z1+fr68gNUTIxV8TBOxcM4FR9jVTyMU/G4Y5z+7MpQIVsEIkkaOXKkBg4cqOuvv1433HCDpk+frpycHOupMwAAYF+2CUR33XWXDh8+rKefflrp6em69tprtWTJkstqkhwAACgftglEkjRs2LDz3iKraN7e3ho/fvxl9+ipOzBWxcM4FQ/jVHyMVfEwTsVzOYyTwxTnWTQAAIArmG2+ugMAAOBCCEQAAMD2CEQAAMD2CEQAAMD2CERu8Prrr6tBgwaqWrWq2rZtq2+++cbdJZWbZ555Rg6Hw2Vp0qSJ1X769GklJCSodu3aqlGjhnr37l3kE8XT0tIUHx+vatWqKSgoSKNHj9bZs2dd+qxatUpt2rSRt7e3IiMjlZiYWBGnd0nWrFmjW265RWFhYXI4HPrss89c2o0xevrppxUaGiofHx/FxsZq9+7dLn2OHj2qvn37ytfXV/7+/hoyZIj1/XiFNm/erA4dOqhq1aqqV6+eJk2aVKSWefPmqUmTJqpatapatmypRYsWlfn5ltafjdOgQYOKvMe6devm0scO4zRx4kT95S9/Uc2aNRUUFKTbb79du3btculTkT9vlfX3XHHGKSYmpsh76sEHH3Tpc6WP08yZM9WqVSvrgxSjo6O1ePFiq/2KfC8ZVKgPP/zQOJ1O8/bbb5tt27aZ+++/3/j7+5tDhw65u7RyMX78eNO8eXNz8OBBazl8+LDV/uCDD5p69eqZ5cuXm40bN5obb7zR/N///Z/VfvbsWdOiRQsTGxtrNm3aZBYtWmQCAwPNuHHjrD4//vijqVatmhk5cqTZvn27efXVV42Hh4dZsmRJhZ5rSS1atMg8+eSTZv78+UaS+fTTT13aX3zxRePn52c+++wz8/3335tbb73VREREmFOnTll9unXrZlq3bm02bNhg1q5dayIjI83dd99ttWdmZprg4GDTt29fs3XrVvPBBx8YHx8f869//cvqs379euPh4WEmTZpktm/fbp566inj5eVltmzZUu5jUBx/Nk4DBw403bp1c3mPHT161KWPHcYpLi7OzJ4922zdutWkpqaaHj16mPr165vs7GyrT0X9vFXm33PFGadOnTqZ+++/3+U9lZmZabXbYZy++OILs3DhQvPDDz+YXbt2mSeeeMJ4eXmZrVu3GmOuzPcSgaiC3XDDDSYhIcFaz8/PN2FhYWbixIlurKr8jB8/3rRu3fq8bcePHzdeXl5m3rx51rYdO3YYSSY5OdkY8/sfwypVqpj09HSrz8yZM42vr6/Jzc01xhgzZswY07x5c5d933XXXSYuLq6Mz6b8nPuHvqCgwISEhJiXX37Z2nb8+HHj7e1tPvjgA2OMMdu3bzeSzLfffmv1Wbx4sXE4HObXX381xhjzxhtvmFq1alljZYwxY8eONY0bN7bW//a3v5n4+HiXetq2bWseeOCBMj3HsnChQHTbbbdd8DV2HCdjjMnIyDCSzOrVq40xFfvzdjn9njt3nIz5PRA9+uijF3yNHcfJGGNq1apl3nzzzSv2vcQtswp05swZpaSkKDY21tpWpUoVxcbGKjk52Y2Vla/du3crLCxMV199tfr27au0tDRJUkpKivLy8lzGo0mTJqpfv741HsnJyWrZsqXLJ4rHxcUpKytL27Zts/r8cR+FfS7nMd23b5/S09NdzsvPz09t27Z1GRt/f39df/31Vp/Y2FhVqVJFX3/9tdWnY8eOcjqdVp+4uDjt2rVLx44ds/pc7uO3atUqBQUFqXHjxnrooYd05MgRq82u45SZmSlJCggIkFRxP2+X2++5c8ep0Ny5cxUYGKgWLVpo3LhxOnnypNVmt3HKz8/Xhx9+qJycHEVHR1+x7yVbfVK1u/3222/Kz88v8nUhwcHB2rlzp5uqKl9t27ZVYmKiGjdurIMHD+rZZ59Vhw4dtHXrVqWnp8vpdBb54tzg4GClp6dLktLT0887XoVtF+uTlZWlU6dOycfHp5zOrvwUntv5zuuP5x0UFOTS7unpqYCAAJc+ERERRfZR2FarVq0Ljl/hPiq7bt26qVevXoqIiNDevXv1xBNPqHv37kpOTpaHh4ctx6mgoEDDhw9Xu3bt1KJFC0mqsJ+3Y8eOXTa/5843TpJ0zz33KDw8XGFhYdq8ebPGjh2rXbt2af78+ZLsM05btmxRdHS0Tp8+rRo1aujTTz9Vs2bNlJqaekW+lwhEKFfdu3e3/t2qVSu1bdtW4eHh+vjjjy/LoILKp0+fPta/W7ZsqVatWqlhw4ZatWqVunTp4sbK3CchIUFbt27VunXr3F1KpXahcRo6dKj175YtWyo0NFRdunTR3r171bBhw4ou020aN26s1NRUZWZm6pNPPtHAgQO1evVqd5dVbrhlVoECAwPl4eFRZCb+oUOHFBIS4qaqKpa/v7+uueYa7dmzRyEhITpz5oyOHz/u0ueP4xESEnLe8Spsu1gfX1/fyzZ0FZ7bxd4rISEhysjIcGk/e/asjh49Wibjd7m+J6+++moFBgZqz549kuw3TsOGDdOCBQu0cuVK1a1b19peUT9vl8vvuQuN0/m0bdtWklzeU3YYJ6fTqcjISEVFRWnixIlq3bq1ZsyYccW+lwhEFcjpdCoqKkrLly+3thUUFGj58uWKjo52Y2UVJzs7W3v37lVoaKiioqLk5eXlMh67du1SWlqaNR7R0dHasmWLyx+0pKQk+fr6qlmzZlafP+6jsM/lPKYREREKCQlxOa+srCx9/fXXLmNz/PhxpaSkWH1WrFihgoIC6xd4dHS01qxZo7y8PKtPUlKSGjdurFq1all9rqTx++WXX3TkyBGFhoZKss84GWM0bNgwffrpp1qxYkWRW4AV9fNW2X/P/dk4nU9qaqokubynrvRxOp+CggLl5uZeue+lMp+mjYv68MMPjbe3t0lMTDTbt283Q4cONf7+/i4z8a8ko0aNMqtWrTL79u0z69evN7GxsSYwMNBkZGQYY35/dLN+/fpmxYoVZuPGjSY6OtpER0dbry98dLNr164mNTXVLFmyxNSpU+e8j26OHj3a7Nixw7z++uuXxWP3J06cMJs2bTKbNm0ykszUqVPNpk2bzP79+40xvz927+/vbz7//HOzefNmc9ttt533sfvrrrvOfP3112bdunWmUaNGLo+THz9+3AQHB5v+/fubrVu3mg8//NBUq1atyOPknp6eZvLkyWbHjh1m/Pjxlepx8ouN04kTJ8xjjz1mkpOTzb59+8yXX35p2rRpYxo1amROnz5t7cMO4/TQQw8ZPz8/s2rVKpfHxU+ePGn1qaift8r8e+7PxmnPnj1mwoQJZuPGjWbfvn3m888/N1dffbXp2LGjtQ87jNPjjz9uVq9ebfbt22c2b95sHn/8ceNwOMyyZcuMMVfme4lA5AavvvqqqV+/vnE6neaGG24wGzZscHdJ5eauu+4yoaGhxul0mquuusrcddddZs+ePVb7qVOnzN///ndTq1YtU61aNXPHHXeYgwcPuuzjp59+Mt27dzc+Pj4mMDDQjBo1yuTl5bn0Wblypbn22muN0+k0V199tZk9e3ZFnN4lWblypZFUZBk4cKAx5vdH7//xj3+Y4OBg4+3tbbp06WJ27drlso8jR46Yu+++29SoUcP4+vqawYMHmxMnTrj0+f7770379u2Nt7e3ueqqq8yLL75YpJaPP/7YXHPNNcbpdJrmzZubhQsXltt5l9TFxunkyZOma9eupk6dOsbLy8uEh4eb+++/v8gvSzuM0/nGSJLLz0JF/rxV1t9zfzZOaWlppmPHjiYgIMB4e3ubyMhIM3r0aJfPITLmyh+ne++914SHhxun02nq1KljunTpYoUhY67M95LDGGPK/roTAADA5YM5RAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRABQCSQmJhb59nAAFYdABKBCDBo0SA6HQw6HQ15eXoqIiNCYMWN0+vTpCquhsoSOBg0aaPr06e4uA8AfeLq7AAD20a1bN82ePVt5eXlKSUnRwIED5XA49NJLL7m7NAA2xxUiABXG29tbISEhqlevnm6//XbFxsYqKSnJas/NzdUjjzyioKAgVa1aVe3bt9e3335rtV9//fWaPHmytX777bfLy8tL2dnZkn7/pnuHw6E9e/aUqr7jx4/rvvvuU506deTr66ubbrpJ33//vdX+zDPP6Nprr9W7776rBg0ayM/PT3369NGJEyesPidOnFDfvn1VvXp1hYaGatq0aYqJidHw4cMlSTExMdq/f79GjBhhXTH7o6VLl6pp06aqUaOGunXrpoMHD5bqXACUDIEIgFts3bpVX331lZxOp7VtzJgx+u9//6s5c+bou+++U2RkpOLi4nT06FFJUqdOnbRq1SpJkjFGa9eulb+/v9atWydJWr16ta666ipFRkaWqqa//vWvysjI0OLFi5WSkqI2bdqoS5cu1vElae/evfrss8+0YMECLViwQKtXr9aLL75otY8cOVLr16/XF198oaSkJK1du1bfffed1T5//nzVrVtXEyZM0MGDB10Cz8mTJzV58mS9++67WrNmjdLS0vTYY4+V6lwAlAyBCECFWbBggWrUqKGqVauqZcuWysjI0OjRoyVJOTk5mjlzpl5++WV1795dzZo103/+8x/5+PjorbfekvT71ZV169YpPz9fmzdvltPpVN++fa2QtGrVKnXq1KlUta1bt07ffPON5s2bp+uvv16NGjXS5MmT5e/vr08++cTqV1BQoMTERLVo0UIdOnRQ//79tXz5ckm/Xx2aM2eOJk+erC5duqhFixaaPXu28vPzrdcHBATIw8NDNWvWVEhIiEJCQqy2vLw8zZo1S9dff73atGmjYcOGWfsGUL6YQwSgwnTu3FkzZ85UTk6Opk2bJk9PT/Xu3VvS71de8vLy1K5dO6u/l5eXbrjhBu3YsUOS1KFDB504cUKbNm3SV199pU6dOikmJsa6QrN69WorYJXU999/r+zsbNWuXdtl+6lTp7R3715rvUGDBqpZs6a1HhoaqoyMDEnSjz/+qLy8PN1www1Wu5+fnxo3blysGqpVq6aGDRued98AyheBCECFqV69unU76+2331br1q311ltvaciQIcV6vb+/v1q3bq1Vq1YpOTlZN998szp27Ki77rpLP/zwg3bv3l3qK0TZ2dkKDQ21rjade9xCXl5eLm0Oh0MFBQWlOua5zrdvY0yZ7BvAxXHLDIBbVKlSRU888YSeeuopnTp1Sg0bNpTT6dT69eutPnl5efr222/VrFkza1unTp20cuVKrVmzRjExMQoICFDTpk31wgsvKDQ0VNdcc02p6mnTpo3S09Pl6empyMhIlyUwMLBY+7j66qvl5eXlMhE8MzNTP/zwg0s/p9PpchsNgPsRiAC4zV//+ld5eHjo9ddfV/Xq1fXQQw9p9OjRWrJkibZv3677779fJ0+edLmCFBMTo6VLl8rT01NNmjSxts2dO7dYV4fy8/OVmprqsuzYsUOxsbGKjo7W7bffrmXLlumnn37SV199pSeffFIbN24s1vnUrFlTAwcO1OjRo7Vy5Upt27ZNQ4YMUZUqVVyeJmvQoIHWrFmjX3/9Vb/99lsJRw1AeSAQAXAbT09PDRs2TJMmTVJOTo5efPFF9e7dW/3791ebNm20Z88eLV26VLVq1bJe06FDBxUUFLiEn5iYGOXn5ysmJuZPj5mdna3rrrvOZbnlllvkcDi0aNEidezYUYMHD9Y111yjPn36aP/+/QoODi72OU2dOlXR0dHq2bOnYmNj1a5dOzVt2lRVq1a1+kyYMEE//fSTGjZsqDp16hR73wDKj8NwgxoAyk1OTo6uuuoqTZkypdhzpQBUPCZVA0AZ2rRpk3bu3KkbbrhBmZmZmjBhgiTptttuc3NlAC6GQAQAZWzy5MnatWuXnE6noqKitHbt2mJPzAbgHtwyAwAAtsekagAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHv/D75SqlpksoZ6AAAAAElFTkSuQmCC",
470 | "text/plain": [
471 | ""
472 | ]
473 | },
474 | "metadata": {},
475 | "output_type": "display_data"
476 | }
477 | ],
478 | "source": [
479 | "plot_histogram(df['row_length'])"
480 | ]
481 | },
482 | {
483 | "cell_type": "code",
484 | "execution_count": 7,
485 | "metadata": {},
486 | "outputs": [
487 | {
488 | "name": "stdout",
489 | "output_type": "stream",
490 | "text": [
491 | "3391\n"
492 | ]
493 | }
494 | ],
495 | "source": [
496 | "# Filter out rows with length more than max_sequence_length (i.e. 11000) to obtain just acceptable length of input\n",
497 | "df_filtered = df[df['row_length'] <= max_sequence_length]\n",
498 | "\n",
499 | "# Display the filtered DataFrame\n",
500 | "print(len(df_filtered))"
501 | ]
502 | },
503 | {
504 | "cell_type": "code",
505 | "execution_count": 8,
506 | "metadata": {},
507 | "outputs": [
508 | {
509 | "data": {
510 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHHCAYAAABeLEexAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFm0lEQVR4nO3deVxV1f7/8fdBRlFAREASkdScZ69GalqSOJWWDRaaer3Z7Uo5dLW8pZVW5pBjptUttbIs+5p5vWZy1cQ5xSmHHMrUVCAHRDARYf3+8MH+dURLEDjgfj0fj/14tNdaZ+/PXoK+22ftcxzGGCMAAAAbc3N1AQAAAK5GIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAIAALZHIAJc4OWXX5bD4SiWc7Vr107t2rWz9r/99ls5HA598cUXxXL+vn37qlq1asVyroJKT0/X3/72N4WGhsrhcGjw4MGuLsl25syZI4fDoS1btri6FNgUgQi4Qbl/kedu3t7eCgsLU0xMjKZNm6Zz584VynmOHz+ul19+Wdu3by+U4xWmklzb9Xj99dc1Z84cPfXUU/roo4/Uu3fva46tVq2a05+3r6+vWrRooQ8//LAYK742h8OhuLg4V5dxTW+//bbmzJnj6jKAPNxdXQBwsxg9erQiIyOVlZWlpKQkffvttxo8eLAmTZqkxYsXq2HDhtbYF198Uc8//3y+jn/8+HG98sorqlatmho3bnzdr1u+fHm+zlMQf1Tbe++9p5ycnCKv4UasXLlSt99+u1566aXrGt+4cWM9++yzkqQTJ07o3//+t/r06aPMzEw98cQTRVlqqff2228rKChIffv2dXUpgBMCEVBIOnXqpObNm1v7I0aM0MqVK9W1a1fdd9992rt3r3x8fCRJ7u7ucncv2l+/8+fPq2zZsvL09CzS8/wZDw8Pl57/eqSkpKhu3brXPf6WW25Rr169rP2+ffvq1ltv1eTJkwlEQCnFW2ZAEbr77rs1cuRIHT58WB9//LHVfrU1RPHx8WrdurUCAgJUrlw51apVS//6178kXV7385e//EWS1K9fP+vtmty3Htq1a6f69esrMTFRd955p8qWLWu99so1RLmys7P1r3/9S6GhofL19dV9992no0ePOo2pVq3aVf9P/vfH/LParraGKCMjQ88++6zCw8Pl5eWlWrVqaeLEiTLGOI3Lfftn0aJFql+/vry8vFSvXj0tW7bs6hN+hZSUFPXv318hISHy9vZWo0aNNHfuXKs/dz3VoUOH9N///teq/eeff76u4+eqVKmSateurR9//DHf1/nAAw+oadOmTq+799575XA4tHjxYqtt06ZNcjgc+vrrr/NV29Xk5ORoypQpqlevnry9vRUSEqInn3xSZ86ccRpXrVo1de3aVWvXrlWLFi3k7e2tW2+99apvD+7cuVNt27aVj4+PqlSpoldffVWzZ892ms9q1app9+7dWr16tTXXV/5sZmZmaujQoapUqZJ8fX11//3369dff3Uas2XLFsXExCgoKEg+Pj6KjIzUX//61xueF9gbd4iAIta7d2/961//0vLly69592D37t3q2rWrGjZsqNGjR8vLy0sHDx7UunXrJEl16tTR6NGjNWrUKA0YMEBt2rSRJN1xxx3WMU6dOqVOnTqpZ8+e6tWrl0JCQv6wrtdee00Oh0PPPfecUlJSNGXKFEVHR2v79u3WnazrcT21/Z4xRvfdd59WrVql/v37q3Hjxvrmm280bNgwHTt2TJMnT3Yav3btWi1cuFD/+Mc/VL58eU2bNk09evTQkSNHVLFixWvW9dtvv6ldu3Y6ePCg4uLiFBkZqQULFqhv375KTU3VoEGDVKdOHX300UcaMmSIqlSpYr0NVqlSpeu+fkm6dOmSfvnlF1WoUCHf19mmTRt99dVXSktLk5+fn4wxWrdundzc3LRmzRrdd999kqQ1a9bIzc1NrVq1yldtV/Pkk09qzpw56tevn5555hkdOnRIb731lrZt26Z169Y53dU7ePCgHnzwQfXv3199+vTRBx98oL59+6pZs2aqV6+eJOnYsWO666675HA4NGLECPn6+urf//63vLy8nM47ZcoUPf300ypXrpxeeOEFScrzc/r000+rQoUKeumll/Tzzz9rypQpiouL02effSbpcsjt0KGDKlWqpOeff14BAQH6+eeftXDhwhueF9icAXBDZs+ebSSZzZs3X3OMv7+/adKkibX/0ksvmd//+k2ePNlIMr/++us1j7F582YjycyePTtPX9u2bY0kM2vWrKv2tW3b1tpftWqVkWRuueUWk5aWZrV//vnnRpKZOnWq1RYREWH69Onzp8f8o9r69OljIiIirP1FixYZSebVV191Gvfggw8ah8NhDh48aLVJMp6enk5tO3bsMJLM9OnT85zr96ZMmWIkmY8//thqu3jxoomKijLlypVzuvaIiAjTpUuXPzze78d26NDB/Prrr+bXX38133//vendu7eRZAYOHJjv68ydu6VLlxpjjNm5c6eRZB566CHTsmVL63X33Xef08/QtVxZx5XWrFljJJl58+Y5tS9btixPe0REhJFkEhISrLaUlBTj5eVlnn32Wavt6aefNg6Hw2zbts1qO3XqlAkMDDSSzKFDh6z2evXqOf3s5Mr9PYqOjjY5OTlW+5AhQ0yZMmVMamqqMcaYL7/88k9/34CC4C0zoBiUK1fuD582CwgIkCR99dVXBV6A7OXlpX79+l33+Mcff1zly5e39h988EFVrlxZS5cuLdD5r9fSpUtVpkwZPfPMM07tzz77rIwxed4Sio6OVvXq1a39hg0bys/PTz/99NOfnic0NFSPPvqo1ebh4aFnnnlG6enpWr16dYGvYfny5apUqZIqVaqkBg0a6KOPPlK/fv00YcKEfF9nkyZNVK5cOSUkJEi6fCeoSpUqevzxx7V161adP39exhitXbvWuvt2IxYsWCB/f3/dc889OnnypLU1a9ZM5cqV06pVq5zG161b1+m8lSpVUq1atZzmf9myZYqKinJaUB8YGKjY2Nh81zdgwACnt5PbtGmj7OxsHT58WNL//11ZsmSJsrKy8n184FoIREAxSE9PdwofV3rkkUfUqlUr/e1vf1NISIh69uypzz//PF/h6JZbbsnXAuqaNWs67TscDtWoUSPf62fy6/DhwwoLC8szH3Xq1LH6f69q1ap5jlGhQoU8612udp6aNWvKzc35r7lrnSc/WrZsqfj4eC1btkwTJ05UQECAzpw54zT/13udZcqUUVRUlNasWSPpciBq06aNWrdurezsbG3cuFF79uzR6dOnCyUQHThwQGfPnlVwcLAV6nK39PR0paSkOI2/nvk/fPiwatSokWfc1dr+zJXny30bMvd8bdu2VY8ePfTKK68oKChI3bp10+zZs5WZmZnvcwG/xxoioIj98ssvOnv27B/+4+Dj46OEhAStWrVK//3vf7Vs2TJ99tlnuvvuu7V8+XKVKVPmT8+Tn3U/1+taHx6ZnZ19XTUVhmudx1yxALs4BQUFKTo6WpIUExOj2rVrq2vXrpo6daqGDh2a7+O1bt1ar732mi5cuKA1a9bohRdeUEBAgOrXr681a9ZY62wKIxDl5OQoODhY8+bNu2r/leuninv+/+x8uR8qunHjRv3nP//RN998o7/+9a968803tXHjRpUrV65I6sLNjztEQBH76KOPJF3+h/OPuLm5qX379po0aZL27Nmj1157TStXrrTewijsT7Y+cOCA074xRgcPHnR6IqxChQpKTU3N89or767kp7aIiAgdP348z1uIP/zwg9VfGCIiInTgwIE8d9kK+zyS1KVLF7Vt21avv/66MjIyrONf73W2adNGFy9e1Keffqpjx45ZwefOO+/UmjVrtGbNGt12221/ulD+elSvXl2nTp1Sq1atFB0dnWdr1KhRvo8ZERGhgwcP5mm/Wlth/Rzffvvteu2117RlyxbNmzdPu3fv1vz58wvl2LAnAhFQhFauXKkxY8YoMjLyD9dTnD59Ok9b7nqM3LcCfH19JemqAaUgPvzwQ6d/rL/44gudOHFCnTp1stqqV6+ujRs36uLFi1bbkiVL8jyen5/aOnfurOzsbL311ltO7ZMnT5bD4XA6/43o3LmzkpKSrKeTpMtPg02fPl3lypVT27ZtC+U8uZ577jmdOnVK7733nnX+673Oli1bysPDQ+PGjVNgYKD19FabNm20ceNGrV69ulDuDknSww8/rOzsbI0ZMyZP36VLlwr08xUTE6MNGzY4fVL56dOnr3oXytfX94Z+hs+cOZPn7tSVvytAQfCWGVBIvv76a/3www+6dOmSkpOTtXLlSsXHxysiIkKLFy+Wt7f3NV87evRoJSQkqEuXLoqIiFBKSorefvttValSRa1bt5Z0OZwEBARo1qxZKl++vHx9fdWyZUtFRkYWqN7AwEC1bt1a/fr1U3JysqZMmaIaNWo4fTTA3/72N33xxRfq2LGjHn74Yf3444/6+OOPnRY557e2e++9V3fddZdeeOEF/fzzz2rUqJGWL1+ur776SoMHD85z7IIaMGCA3nnnHfXt21eJiYmqVq2avvjiC61bt05Tpkz5wzVdBdGpUyfVr19fkyZN0sCBA/N1nWXLllWzZs20ceNG6zOIpMt3iDIyMpSRkZGvQLRlyxa9+uqredrbtWuntm3b6sknn9TYsWO1fft2dejQQR4eHjpw4IAWLFigqVOn6sEHH8zXtQ8fPlwff/yx7rnnHj399NPWY/dVq1bV6dOnne4KNWvWTDNnztSrr76qGjVqKDg4WHffffd1n2vu3Ll6++23df/996t69eo6d+6c3nvvPfn5+alz5875qhtw4roH3ICbQ+7jwrmbp6enCQ0NNffcc4+ZOnWq0+Pdua587H7FihWmW7duJiwszHh6epqwsDDz6KOPmv379zu97quvvjJ169Y17u7uTo+5t23b1tSrV++q9V3rsftPP/3UjBgxwgQHBxsfHx/TpUsXc/jw4Tyvf/PNN80tt9xivLy8TKtWrcyWLVvyHPOParvysXtjjDl37pwZMmSICQsLMx4eHqZmzZpmwoQJTo9bG3PtR8iv9XEAV0pOTjb9+vUzQUFBxtPT0zRo0OCqHw2Q38furzV2zpw5Ttd+vddpjDHDhg0zksy4ceOc2mvUqGEkmR9//PG66vv9z+KV25gxY6xx7777rmnWrJnx8fEx5cuXNw0aNDDDhw83x48f/9Nrvdqf/7Zt20ybNm2Ml5eXqVKlihk7dqyZNm2akWSSkpKscUlJSaZLly6mfPnyRpJ1nGt9fEXuz+uqVauMMcZs3brVPProo6Zq1arGy8vLBAcHm65du5otW7Zc1/wA1+IwxoUrEwEAN63BgwfrnXfeUXp6erEtwgcKijVEAIAb9ttvvzntnzp1Sh999JFat25NGEKpwBoiAMANi4qKUrt27VSnTh0lJyfr/fffV1pamkaOHOnq0oDrQiACANywzp0764svvtC7774rh8Ohpk2b6v3339edd97p6tKA68IaIgAAYHusIQIAALZHIAIAALbHGqLrkJOTo+PHj6t8+fKF/vUJAACgaBhjdO7cOYWFheX5oucrEYiuw/HjxxUeHu7qMgAAQAEcPXpUVapU+cMxBKLrkPsR/0ePHpWfn5+LqwEAANcjLS1N4eHh1/VVPQSi65D7Npmfnx+BCACAUuZ6lruwqBoAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANgegQgAANieu6sLgHTkyBGdPHnS1WXkS1BQkKpWrerqMgAAKBQEIhc7cuSIatWuowu/nXd1Kfni7VNW+37YSygCANwUCEQudvLkSV347bwqdn1WHhXDXV3Odck6dVSnlrypkydPEogAADcFAlEJ4VExXF6hNVxdBgAAtsSiagAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsEIgAAYHsuDUQJCQm69957FRYWJofDoUWLFll9WVlZeu6559SgQQP5+voqLCxMjz/+uI4fP+50jNOnTys2NlZ+fn4KCAhQ//79lZ6e7jRm586datOmjby9vRUeHq7x48cXx+UBAIBSwqWBKCMjQ40aNdKMGTPy9J0/f15bt27VyJEjtXXrVi1cuFD79u3Tfffd5zQuNjZWu3fvVnx8vJYsWaKEhAQNGDDA6k9LS1OHDh0UERGhxMRETZgwQS+//LLefffdIr8+AABQOri78uSdOnVSp06drtrn7++v+Ph4p7a33npLLVq00JEjR1S1alXt3btXy5Yt0+bNm9W8eXNJ0vTp09W5c2dNnDhRYWFhmjdvni5evKgPPvhAnp6eqlevnrZv365JkyY5BScAAGBfpWoN0dmzZ+VwOBQQECBJ2rBhgwICAqwwJEnR0dFyc3PTpk2brDF33nmnPD09rTExMTHat2+fzpw5c9XzZGZmKi0tzWkDAAA3r1ITiC5cuKDnnntOjz76qPz8/CRJSUlJCg4Odhrn7u6uwMBAJSUlWWNCQkKcxuTu54650tixY+Xv729t4eHhhX05AACgBCkVgSgrK0sPP/ywjDGaOXNmkZ9vxIgROnv2rLUdPXq0yM8JAABcx6VriK5Hbhg6fPiwVq5cad0dkqTQ0FClpKQ4jb906ZJOnz6t0NBQa0xycrLTmNz93DFX8vLykpeXV2FeBgAAKMFK9B2i3DB04MAB/e9//1PFihWd+qOiopSamqrExESrbeXKlcrJyVHLli2tMQkJCcrKyrLGxMfHq1atWqpQoULxXAgAACjRXHqHKD09XQcPHrT2Dx06pO3btyswMFCVK1fWgw8+qK1bt2rJkiXKzs621vwEBgbK09NTderUUceOHfXEE09o1qxZysrKUlxcnHr27KmwsDBJ0mOPPaZXXnlF/fv313PPPaddu3Zp6tSpmjx5skuu+Wayd+9eV5eQL0FBQapataqrywAAlEAuDURbtmzRXXfdZe0PHTpUktSnTx+9/PLLWrx4sSSpcePGTq9btWqV2rVrJ0maN2+e4uLi1L59e7m5ualHjx6aNm2aNdbf31/Lly/XwIED1axZMwUFBWnUqFE8cn8DstPPSA6HevXq5epS8sXbp6z2/bCXUAQAyMOlgahdu3Yyxlyz/4/6cgUGBuqTTz75wzENGzbUmjVr8l0fri4nM10yRhW7PiuPiqXjCbysU0d1asmbOnnyJIEIAJBHiV9UjZLLo2K4vEJruLoMAABuWIleVA0AAFAcCEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2CEQAAMD2XBqIEhISdO+99yosLEwOh0OLFi1y6jfGaNSoUapcubJ8fHwUHR2tAwcOOI05ffq0YmNj5efnp4CAAPXv31/p6elOY3bu3Kk2bdrI29tb4eHhGj9+fFFfGgAAKEVcGogyMjLUqFEjzZgx46r948eP17Rp0zRr1ixt2rRJvr6+iomJ0YULF6wxsbGx2r17t+Lj47VkyRIlJCRowIABVn9aWpo6dOigiIgIJSYmasKECXr55Zf17rvvFvn1AQCA0sHdlSfv1KmTOnXqdNU+Y4ymTJmiF198Ud26dZMkffjhhwoJCdGiRYvUs2dP7d27V8uWLdPmzZvVvHlzSdL06dPVuXNnTZw4UWFhYZo3b54uXryoDz74QJ6enqpXr562b9+uSZMmOQUnAABgXyV2DdGhQ4eUlJSk6Ohoq83f318tW7bUhg0bJEkbNmxQQECAFYYkKTo6Wm5ubtq0aZM15s4775Snp6c1JiYmRvv27dOZM2eueu7MzEylpaU5bQAA4OZVYgNRUlKSJCkkJMSpPSQkxOpLSkpScHCwU7+7u7sCAwOdxlztGL8/x5XGjh0rf39/awsPD7/xCwIAACVWiQ1ErjRixAidPXvW2o4ePerqkgAAQBEqsYEoNDRUkpScnOzUnpycbPWFhoYqJSXFqf/SpUs6ffq005irHeP357iSl5eX/Pz8nDYAAHDzKrGBKDIyUqGhoVqxYoXVlpaWpk2bNikqKkqSFBUVpdTUVCUmJlpjVq5cqZycHLVs2dIak5CQoKysLGtMfHy8atWqpQoVKhTT1QAAgJLMpYEoPT1d27dv1/bt2yVdXki9fft2HTlyRA6HQ4MHD9arr76qxYsX6/vvv9fjjz+usLAwde/eXZJUp04ddezYUU888YS+++47rVu3TnFxcerZs6fCwsIkSY899pg8PT3Vv39/7d69W5999pmmTp2qoUOHuuiqAQBASePSx+63bNmiu+66y9rPDSl9+vTRnDlzNHz4cGVkZGjAgAFKTU1V69attWzZMnl7e1uvmTdvnuLi4tS+fXu5ubmpR48emjZtmtXv7++v5cuXa+DAgWrWrJmCgoI0atQoHrkHAAAWlwaidu3ayRhzzX6Hw6HRo0dr9OjR1xwTGBioTz755A/P07BhQ61Zs6bAdQIAgJtbiV1DBAAAUFwIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYIRAAAwPYKFIh++umnwq7jqrKzszVy5EhFRkbKx8dH1atX15gxY2SMscYYYzRq1ChVrlxZPj4+io6O1oEDB5yOc/r0acXGxsrPz08BAQHq37+/0tPTi+UaAABAyVegQFSjRg3ddddd+vjjj3XhwoXCrskybtw4zZw5U2+99Zb27t2rcePGafz48Zo+fbo1Zvz48Zo2bZpmzZqlTZs2ydfXVzExMU51xcbGavfu3YqPj9eSJUuUkJCgAQMGFFndAACgdClQINq6dasaNmyooUOHKjQ0VE8++aS+++67wq5N69evV7du3dSlSxdVq1ZNDz74oDp06GCdyxijKVOm6MUXX1S3bt3UsGFDffjhhzp+/LgWLVokSdq7d6+WLVumf//732rZsqVat26t6dOna/78+Tp+/Hih1wwAAEqfAgWixo0ba+rUqTp+/Lg++OADnThxQq1bt1b9+vU1adIk/frrr4VS3B133KEVK1Zo//79kqQdO3Zo7dq16tSpkyTp0KFDSkpKUnR0tPUaf39/tWzZUhs2bJAkbdiwQQEBAWrevLk1Jjo6Wm5ubtq0aVOh1AkAAEq3G1pU7e7urgceeEALFizQuHHjdPDgQf3zn/9UeHi4Hn/8cZ04ceKGinv++efVs2dP1a5dWx4eHmrSpIkGDx6s2NhYSVJSUpIkKSQkxOl1ISEhVl9SUpKCg4Pz1B0YGGiNuVJmZqbS0tKcNgAAcPO6oUC0ZcsW/eMf/1DlypU1adIk/fOf/9SPP/6o+Ph4HT9+XN26dbuh4j7//HPNmzdPn3zyibZu3aq5c+dq4sSJmjt37g0d98+MHTtW/v7+1hYeHl6k5wMAAK7lXpAXTZo0SbNnz9a+ffvUuXNnffjhh+rcubPc3C7nq8jISM2ZM0fVqlW7oeKGDRtm3SWSpAYNGujw4cMaO3as+vTpo9DQUElScnKyKleubL0uOTlZjRs3liSFhoYqJSXF6biXLl3S6dOnrddfacSIERo6dKi1n5aWRigCAOAmVqA7RDNnztRjjz2mw4cPa9GiReratasVhnIFBwfr/fffv6Hizp8/n+e4ZcqUUU5OjqTLwSs0NFQrVqyw+tPS0rRp0yZFRUVJkqKiopSamqrExERrzMqVK5WTk6OWLVte9bxeXl7y8/Nz2gAAwM2rQHeIrvycn6vx9PRUnz59CnJ4y7333qvXXntNVatWVb169bRt2zZNmjRJf/3rXyVJDodDgwcP1quvvqqaNWsqMjJSI0eOVFhYmLp37y5JqlOnjjp27KgnnnhCs2bNUlZWluLi4tSzZ0+FhYXdUH0AAODmUKBANHv2bJUrV04PPfSQU/uCBQt0/vz5Gw5CuaZPn66RI0fqH//4h1JSUhQWFqYnn3xSo0aNssYMHz5cGRkZGjBggFJTU9W6dWstW7ZM3t7e1ph58+YpLi5O7du3l5ubm3r06KFp06YVSo0AAKD0K1AgGjt2rN5555087cHBwRowYEChBaLy5ctrypQpmjJlyjXHOBwOjR49WqNHj77mmMDAQH3yySeFUhMAALj5FGgN0ZEjRxQZGZmnPSIiQkeOHLnhogAAAIpTgQJRcHCwdu7cmad9x44dqlix4g0XBQAAUJwKFIgeffRRPfPMM1q1apWys7OVnZ2tlStXatCgQdYj8gAAAKVFgdYQjRkzRj///LPat28vd/fLh8jJydHjjz+u119/vVALBAAAKGoFCkSenp767LPPNGbMGO3YsUM+Pj5q0KCBIiIiCrs+AACAIlegQJTrtttu02233VZYtQAAALhEgQJRdna25syZoxUrViglJcX65OhcK1euLJTiAAAAikOBAtGgQYM0Z84cdenSRfXr15fD4SjsugAAAIpNgQLR/Pnz9fnnn6tz586FXQ8AAECxK9Bj956enqpRo0Zh1wIAAOASBQpEzz77rKZOnSpjTGHXAwAAUOwK9JbZ2rVrtWrVKn399deqV6+ePDw8nPoXLlxYKMUBAAAUhwIFooCAAN1///2FXQsAAIBLFCgQzZ49u7DrAAAAcJkCrSGSpEuXLul///uf3nnnHZ07d06SdPz4caWnpxdacQAAAMWhQHeIDh8+rI4dO+rIkSPKzMzUPffco/Lly2vcuHHKzMzUrFmzCrtOAACAIlOgO0SDBg1S8+bNdebMGfn4+Fjt999/v1asWFFoxQEAABSHAt0hWrNmjdavXy9PT0+n9mrVqunYsWOFUhgAAEBxKdAdopycHGVnZ+dp/+WXX1S+fPkbLgoAAKA4FSgQdejQQVOmTLH2HQ6H0tPT9dJLL/F1HgAAoNQp0Ftmb775pmJiYlS3bl1duHBBjz32mA4cOKCgoCB9+umnhV0jAABAkSpQIKpSpYp27Nih+fPna+fOnUpPT1f//v0VGxvrtMgaAACgNChQIJIkd3d39erVqzBrAQAAcIkCBaIPP/zwD/sff/zxAhUDAADgCgUKRIMGDXLaz8rK0vnz5+Xp6amyZcsSiAAAQKlSoKfMzpw547Slp6dr3759at26NYuqAQBAqVPg7zK7Us2aNfXGG2/kuXsEAABQ0hVaIJIuL7Q+fvx4YR4SAACgyBVoDdHixYud9o0xOnHihN566y21atWqUAoDAAAoLgUKRN27d3fadzgcqlSpku6++269+eabhVEXAABAsSlQIMrJySnsOgAAAFymUNcQAQAAlEYFukM0dOjQ6x47adKkgpwCAACg2BQoEG3btk3btm1TVlaWatWqJUnav3+/ypQpo6ZNm1rjHA5H4VQJAABQhAoUiO69916VL19ec+fOVYUKFSRd/rDGfv36qU2bNnr22WcLtUgAAICiVKA1RG+++abGjh1rhSFJqlChgl599VWeMgMAAKVOgQJRWlqafv311zztv/76q86dO3fDRQEAABSnAgWi+++/X/369dPChQv1yy+/6JdfftH//d//qX///nrggQcKu0YAAIAiVaA1RLNmzdI///lPPfbYY8rKyrp8IHd39e/fXxMmTCjUAgEAAIpagQJR2bJl9fbbb2vChAn68ccfJUnVq1eXr69voRYHAABQHG7ogxlPnDihEydOqGbNmvL19ZUxprDqAgAAKDYFCkSnTp1S+/btddttt6lz5846ceKEJKl///48cg8AAEqdAgWiIUOGyMPDQ0eOHFHZsmWt9kceeUTLli0rtOIAAACKQ4EC0fLlyzVu3DhVqVLFqb1mzZo6fPhwoRSW69ixY+rVq5cqVqwoHx8fNWjQQFu2bLH6jTEaNWqUKleuLB8fH0VHR+vAgQNOxzh9+rRiY2Pl5+engIAA9e/fX+np6YVaJwAAKL0KFIgyMjKc7gzlOn36tLy8vG64qFxnzpxRq1at5OHhoa+//lp79uzRm2++6fSBkOPHj9e0adM0a9Ysbdq0Sb6+voqJidGFCxesMbGxsdq9e7fi4+O1ZMkSJSQkaMCAAYVWJwAAKN0K9JRZmzZt9OGHH2rMmDGSLn9nWU5OjsaPH6+77rqr0IobN26cwsPDNXv2bKstMjLS+m9jjKZMmaIXX3xR3bp1kyR9+OGHCgkJ0aJFi9SzZ0/t3btXy5Yt0+bNm9W8eXNJ0vTp09W5c2dNnDhRYWFhhVYvAAAonQp0h2j8+PF699131alTJ128eFHDhw9X/fr1lZCQoHHjxhVacYsXL1bz5s310EMPKTg4WE2aNNF7771n9R86dEhJSUmKjo622vz9/dWyZUtt2LBBkrRhwwYFBARYYUiSoqOj5ebmpk2bNl31vJmZmUpLS3PaAADAzatAgah+/frav3+/WrdurW7duikjI0MPPPCAtm3bpurVqxdacT/99JNmzpypmjVr6ptvvtFTTz2lZ555RnPnzpUkJSUlSZJCQkKcXhcSEmL1JSUlKTg42Knf3d1dgYGB1pgrjR07Vv7+/tYWHh5eaNcEAABKnny/ZZaVlaWOHTtq1qxZeuGFF4qiJktOTo6aN2+u119/XZLUpEkT7dq1S7NmzVKfPn2K7LwjRozQ0KFDrf20tDRCEQAAN7F83yHy8PDQzp07i6KWPCpXrqy6des6tdWpU0dHjhyRJIWGhkqSkpOTncYkJydbfaGhoUpJSXHqv3Tpkk6fPm2NuZKXl5f8/PycNgAAcPMq0FtmvXr10vvvv1/YteTRqlUr7du3z6lt//79ioiIkHR5gXVoaKhWrFhh9aelpWnTpk2KioqSJEVFRSk1NVWJiYnWmJUrVyonJ0ctW7Ys8msAAAAlX4GeMrt06ZI++OAD/e9//1OzZs3yfIfZpEmTCqW4IUOG6I477tDrr7+uhx9+WN99953effddvfvuu5IuP902ePBgvfrqq6pZs6YiIyM1cuRIhYWFqXv37pIu31Hq2LGjnnjiCc2aNUtZWVmKi4tTz549ecIMAABIymcg+umnn1StWjXt2rVLTZs2lXT5js3vORyOQivuL3/5i7788kuNGDFCo0ePVmRkpKZMmaLY2FhrzPDhw5WRkaEBAwYoNTVVrVu31rJly+Tt7W2NmTdvnuLi4tS+fXu5ubmpR48emjZtWqHVCQAASrd8BaKaNWvqxIkTWrVqlaTLX9Uxbdq0PE95FaauXbuqa9eu1+x3OBwaPXq0Ro8efc0xgYGB+uSTT4qiPAAAcBPI1xqiK7/N/uuvv1ZGRkahFgQAAFDcCrSoOteVAQkAAKA0ylcgcjgcedYIFeaaIQAAAFfI1xoiY4z69u1rfYHrhQsX9Pe//z3PU2YLFy4svAoBAACKWL4C0ZWfDt2rV69CLQYAAMAV8hWIfv+t8wAAADeLG1pUDQAAcDMgEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANsjEAEAANtzd3UBQHHau3evq0vIl6CgIFWtWtXVZQDATY9ABFvITj8jORzq1auXq0vJF2+fstr3w15CEQAUMQIRbCEnM10yRhW7PiuPiuGuLue6ZJ06qlNL3tTJkycJRABQxAhEsBWPiuHyCq3h6jIAACUMi6oBAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtEYgAAIDtlapA9MYbb8jhcGjw4MFW24ULFzRw4EBVrFhR5cqVU48ePZScnOz0uiNHjqhLly4qW7asgoODNWzYMF26dKmYqwcAACVVqQlEmzdv1jvvvKOGDRs6tQ8ZMkT/+c9/tGDBAq1evVrHjx/XAw88YPVnZ2erS5cuunjxotavX6+5c+dqzpw5GjVqVHFfAgAAKKFKRSBKT09XbGys3nvvPVWoUMFqP3v2rN5//31NmjRJd999t5o1a6bZs2dr/fr12rhxoyRp+fLl2rNnjz7++GM1btxYnTp10pgxYzRjxgxdvHjRVZcEAABKkFIRiAYOHKguXbooOjraqT0xMVFZWVlO7bVr11bVqlW1YcMGSdKGDRvUoEEDhYSEWGNiYmKUlpam3bt3X/V8mZmZSktLc9oAAMDNq8R/l9n8+fO1detWbd68OU9fUlKSPD09FRAQ4NQeEhKipKQka8zvw1Buf27f1YwdO1avvPJKIVQPAABKgxJ9h+jo0aMaNGiQ5s2bJ29v72I774gRI3T27FlrO3r0aLGdGwAAFL8SHYgSExOVkpKipk2byt3dXe7u7lq9erWmTZsmd3d3hYSE6OLFi0pNTXV6XXJyskJDQyVJoaGheZ46y93PHXMlLy8v+fn5OW0AAODmVaIDUfv27fX9999r+/bt1ta8eXPFxsZa/+3h4aEVK1ZYr9m3b5+OHDmiqKgoSVJUVJS+//57paSkWGPi4+Pl5+enunXrFvs1AQCAkqdEryEqX7686tev79Tm6+urihUrWu39+/fX0KFDFRgYKD8/Pz399NOKiorS7bffLknq0KGD6tatq969e2v8+PFKSkrSiy++qIEDB8rLy6vYrwkAAJQ8JToQXY/JkyfLzc1NPXr0UGZmpmJiYvT2229b/WXKlNGSJUv01FNPKSoqSr6+vurTp49Gjx7twqoBAEBJUuoC0bfffuu07+3trRkzZmjGjBnXfE1ERISWLl1axJUBAIDSqkSvIQIAACgOBCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7BCIAAGB7JToQjR07Vn/5y19Uvnx5BQcHq3v37tq3b5/TmAsXLmjgwIGqWLGiypUrpx49eig5OdlpzJEjR9SlSxeVLVtWwcHBGjZsmC5dulSclwIAAEqwEh2IVq9erYEDB2rjxo2Kj49XVlaWOnTooIyMDGvMkCFD9J///EcLFizQ6tWrdfz4cT3wwANWf3Z2trp06aKLFy9q/fr1mjt3rubMmaNRo0a54pIAAEAJ5O7qAv7IsmXLnPbnzJmj4OBgJSYm6s4779TZs2f1/vvv65NPPtHdd98tSZo9e7bq1KmjjRs36vbbb9fy5cu1Z88e/e9//1NISIgaN26sMWPG6LnnntPLL78sT09PV1waAAAoQUp0ILrS2bNnJUmBgYGSpMTERGVlZSk6OtoaU7t2bVWtWlUbNmzQ7bffrg0bNqhBgwYKCQmxxsTExOipp57S7t271aRJkzznyczMVGZmprWflpZWVJcE/Km9e/e6uoR8CQoKUtWqVV1dBgDkS6kJRDk5ORo8eLBatWql+vXrS5KSkpLk6empgIAAp7EhISFKSkqyxvw+DOX25/ZdzdixY/XKK68U8hUA+ZOdfkZyONSrVy9Xl5Iv3j5lte+HvYQiAKVKqQlEAwcO1K5du7R27doiP9eIESM0dOhQaz8tLU3h4eFFfl7g93Iy0yVjVLHrs/KoWDp+/rJOHdWpJW/q5MmTBCIApUqpCERxcXFasmSJEhISVKVKFas9NDRUFy9eVGpqqtNdouTkZIWGhlpjvvvuO6fj5T6FljvmSl5eXvLy8irkqwAKxqNiuLxCa7i6DAC4qZXop8yMMYqLi9OXX36plStXKjIy0qm/WbNm8vDw0IoVK6y2ffv26ciRI4qKipIkRUVF6fvvv1dKSoo1Jj4+Xn5+fqpbt27xXAgAACjRSvQdooEDB+qTTz7RV199pfLly1trfvz9/eXj4yN/f3/1799fQ4cOVWBgoPz8/PT0008rKipKt99+uySpQ4cOqlu3rnr37q3x48crKSlJL774ogYOHMhdIAAAIKmEB6KZM2dKktq1a+fUPnv2bPXt21eSNHnyZLm5ualHjx7KzMxUTEyM3n77bWtsmTJltGTJEj311FOKioqSr6+v+vTpo9GjRxfXZQAAgBKuRAciY8yfjvH29taMGTM0Y8aMa46JiIjQ0qVLC7M0AABwEynRa4gAAACKA4EIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYHoEIAADYnrurCwBw89m7d6+rS8iXoKAgVa1a1dVlAHAhAhGAQpOdfkZyONSrVy9Xl5Iv3j5lte+HvYQiwMYIRAAKTU5mumSMKnZ9Vh4Vw11dznXJOnVUp5a8qZMnTxKIABsjEAEodB4Vw+UVWsPVZQDAdWNRNQAAsD0CEQAAsD0CEQAAsD0CEQAAsD0CEQAAsD2eMgMA8WGSgN0RiADYGh8mCUAiEAGwOT5MEoBEIAIASXyYJGB3BCIAKKVK27onibVPKLkIRABQypTWdU8Sa59QchGIAKCUKY3rnqT/v/ZpzZo1qlOnjqvLuW7c1bIHAhEAlFKlbd1Tab2zxV0teyAQAQCKRWm8s8UTffZBIAIAFKvSdmcL9mCrQDRjxgxNmDBBSUlJatSokaZPn64WLVq4uiwAQAlX2p7oY91T/tkmEH322WcaOnSoZs2apZYtW2rKlCmKiYnRvn37FBwc7OryAAAlUGld9+Tl5a3/+78vVLlyZVeXct1cHeJsE4gmTZqkJ554Qv369ZMkzZo1S//973/1wQcf6Pnnn3dxdQCAkqg0rnu68Mtupa78t7p27erqUvLF1YvXbRGILl68qMTERI0YMcJqc3NzU3R0tDZs2ODCygAApUFpWveUdepoqQtxJWHxui0C0cmTJ5Wdna2QkBCn9pCQEP3www95xmdmZiozM9PaP3v2rCQpLS2t0GtLT0+/fM6kg8q5eKHQj18Usk4dlUTNRY2aiwc1F5/SWHdprjknK7PU1JyTdfnf3PT09EL9tzb3WMaYPx9sbODYsWNGklm/fr1T+7Bhw0yLFi3yjH/ppZeMJDY2NjY2NrabYDt69OifZgVb3CEKCgpSmTJllJyc7NSenJys0NDQPONHjBihoUOHWvs5OTk6ffq0KlasKIfDka9zp6WlKTw8XEePHpWfn1/BLgB5MK9Fg3ktfMxp0WBei8bNNq/GGJ07d05hYWF/OtYWgcjT01PNmjXTihUr1L17d0mXQ86KFSsUFxeXZ7yXl5e8vLyc2gICAm6oBj8/v5vih6ukYV6LBvNa+JjTosG8Fo2baV79/f2va5wtApEkDR06VH369FHz5s3VokULTZkyRRkZGdZTZwAAwL5sE4geeeQR/frrrxo1apSSkpLUuHFjLVu2LM9CawAAYD+2CUSSFBcXd9W3yIqSl5eXXnrppTxvweHGMK9Fg3ktfMxp0WBei4ad59VhzPU8iwYAAHDzcnN1AQAAAK5GIAIAALZHIAIAALZHIAIAALZHICpiM2bMULVq1eTt7a2WLVvqu+++c3VJJcbYsWP1l7/8ReXLl1dwcLC6d++uffv2OY25cOGCBg4cqIoVK6pcuXLq0aNHnk8cP3LkiLp06aKyZcsqODhYw4YN06VLl5zGfPvtt2ratKm8vLxUo0YNzZkzp6gvr0R444035HA4NHjwYKuNOS2YY8eOqVevXqpYsaJ8fHzUoEEDbdmyxeo3xmjUqFGqXLmyfHx8FB0drQMHDjgd4/Tp04qNjZWfn58CAgLUv39/6/sMc+3cuVNt2rSRt7e3wsPDNX78+GK5vuKWnZ2tkSNHKjIyUj4+PqpevbrGjBnj9J1TzOmfS0hI0L333quwsDA5HA4tWrTIqb8453DBggWqXbu2vL291aBBAy1durTQr7dI3fg3heFa5s+fbzw9Pc0HH3xgdu/ebZ544gkTEBBgkpOTXV1aiRATE2Nmz55tdu3aZbZv3246d+5sqlatatLT060xf//73014eLhZsWKF2bJli7n99tvNHXfcYfVfunTJ1K9f30RHR5tt27aZpUuXmqCgIDNixAhrzE8//WTKli1rhg4davbs2WOmT59uypQpY5YtW1as11vcvvvuO1OtWjXTsGFDM2jQIKudOc2/06dPm4iICNO3b1+zadMm89NPP5lvvvnGHDx40BrzxhtvGH9/f7No0SKzY8cOc99995nIyEjz22+/WWM6duxoGjVqZDZu3GjWrFljatSoYR599FGr/+zZsyYkJMTExsaaXbt2mU8//dT4+PiYd955p1ivtzi89tprpmLFimbJkiXm0KFDZsGCBaZcuXJm6tSp1hjm9M8tXbrUvPDCC2bhwoVGkvnyyy+d+otrDtetW2fKlCljxo8fb/bs2WNefPFF4+HhYb7//vsin4PCQiAqQi1atDADBw609rOzs01YWJgZO3asC6squVJSUowks3r1amOMMampqcbDw8MsWLDAGrN3714jyWzYsMEYc/kvAzc3N5OUlGSNmTlzpvHz8zOZmZnGGGOGDx9u6tWr53SuRx55xMTExBT1JbnMuXPnTM2aNU18fLxp27atFYiY04J57rnnTOvWra/Zn5OTY0JDQ82ECROsttTUVOPl5WU+/fRTY4wxe/bsMZLM5s2brTFff/21cTgc5tixY8YYY95++21ToUIFa55zz12rVq3CviSX69Kli/nrX//q1PbAAw+Y2NhYYwxzWhBXBqLinMOHH37YdOnSxameli1bmieffLJQr7Eo8ZZZEbl48aISExMVHR1ttbm5uSk6OlobNmxwYWUl19mzZyVJgYGBkqTExERlZWU5zWHt2rVVtWpVaw43bNigBg0aOH3ieExMjNLS0rR7925rzO+PkTvmZv5zGDhwoLp06ZLnupnTglm8eLGaN2+uhx56SMHBwWrSpInee+89q//QoUNKSkpymhN/f3+1bNnSaV4DAgLUvHlza0x0dLTc3Ny0adMma8ydd94pT09Pa0xMTIz27dunM2fOFPVlFqs77rhDK1as0P79+yVJO3bs0Nq1a9WpUydJzGlhKM45vBn+TiAQFZGTJ08qOzs7z1eDhISEKCkpyUVVlVw5OTkaPHiwWrVqpfr160uSkpKS5OnpmeeLdX8/h0lJSVed49y+PxqTlpam3377rSgux6Xmz5+vrVu3auzYsXn6mNOC+emnnzRz5kzVrFlT33zzjZ566ik988wzmjt3rqT/Py9/9PuelJSk4OBgp353d3cFBgbma+5vFs8//7x69uyp2rVry8PDQ02aNNHgwYMVGxsriTktDMU5h9caU5rm2FZf3YGSa+DAgdq1a5fWrl3r6lJKtaNHj2rQoEGKj4+Xt7e3q8u5aeTk5Kh58+Z6/fXXJUlNmjTRrl27NGvWLPXp08fF1ZVOn3/+uebNm6dPPvlE9erV0/bt2zV48GCFhYUxp3AJ7hAVkaCgIJUpUybP0zvJyckKDQ11UVUlU1xcnJYsWaJVq1apSpUqVntoaKguXryo1NRUp/G/n8PQ0NCrznFu3x+N8fPzk4+PT2FfjkslJiYqJSVFTZs2lbu7u9zd3bV69WpNmzZN7u7uCgkJYU4LoHLlyqpbt65TW506dXTkyBFJ/39e/uj3PTQ0VCkpKU79ly5d0unTp/M19zeLYcOGWXeJGjRooN69e2vIkCHWnU3m9MYV5xxea0xpmmMCURHx9PRUs2bNtGLFCqstJydHK1asUFRUlAsrKzmMMYqLi9OXX36plStXKjIy0qm/WbNm8vDwcJrDffv26ciRI9YcRkVF6fvvv3f6hY6Pj5efn5/1D1hUVJTTMXLH3Ix/Du3bt9f333+v7du3W1vz5s0VGxtr/Tdzmn+tWrXK85EQ+/fvV0REhCQpMjJSoaGhTnOSlpamTZs2Oc1ramqqEhMTrTErV65UTk6OWrZsaY1JSEhQVlaWNSY+Pl61atVShQoViuz6XOH8+fNyc3P+J6hMmTLKycmRxJwWhuKcw5vi7wRXr+q+mc2fP994eXmZOXPmmD179pgBAwaYgIAAp6d37Oypp54y/v7+5ttvvzUnTpywtvPnz1tj/v73v5uqVaualStXmi1btpioqCgTFRVl9ec+It6hQwezfft2s2zZMlOpUqWrPiI+bNgws3fvXjNjxoyb+hHxK/3+KTNjmNOC+O6774y7u7t57bXXzIEDB8y8efNM2bJlzccff2yNeeONN0xAQID56quvzM6dO023bt2u+nhzkyZNzKZNm8zatWtNzZo1nR5vTk1NNSEhIaZ3795m165dZv78+aZs2bI3zSPiv9enTx9zyy23WI/dL1y40AQFBZnhw4dbY5jTP3fu3Dmzbds2s23bNiPJTJo0yWzbts0cPnzYGFN8c7hu3Trj7u5uJk6caPbu3WteeuklHruHs+nTp5uqVasaT09P06JFC7Nx40ZXl1RiSLrqNnv2bGvMb7/9Zv7xj3+YChUqmLJly5r777/fnDhxwuk4P//8s+nUqZPx8fExQUFB5tlnnzVZWVlOY1atWmUaN25sPD09za233up0jpvdlYGIOS2Y//znP6Z+/frGy8vL1K5d27z77rtO/Tk5OWbkyJEmJCTEeHl5mfbt25t9+/Y5jTl16pR59NFHTbly5Yyfn5/p16+fOXfunNOYHTt2mNatWxsvLy9zyy23mDfeeKPIr80V0tLSzKBBg0zVqlWNt7e3ufXWW80LL7zg9Gg3c/rnVq1addW/R/v06WOMKd45/Pzzz81tt91mPD09Tb169cx///vfIrvuouAw5ncfCwoAAGBDrCECAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACAAC2RyACgBJgzpw5CggIcHUZgG0RiAAUi759+8rhcMjhcMjDw0ORkZEaPny4Lly4UGw1lJTQUa1aNU2ZMsXVZQD4HXdXFwDAPjp27KjZs2crKytLiYmJ6tOnjxwOh8aNG+fq0gDYHHeIABQbLy8vhYaGKjw8XN27d1d0dLTi4+Ot/szMTD3zzDMKDg6Wt7e3Wrdurc2bN1v9zZs318SJE6397t27y8PDQ+np6ZKkX375RQ6HQwcPHixQfampqfrb3/6mSpUqyc/PT3fffbd27Nhh9b/88stq3LixPvroI1WrVk3+/v7q2bOnzp07Z405d+6cYmNj5evrq8qVK2vy5Mlq166dBg8eLElq166dDh8+rCFDhlh3zH7vm2++UZ06dVSuXDl17NhRJ06cKNC1AMgfAhEAl9i1a5fWr18vT09Pq2348OH6v//7P82dO1dbt25VjRo1FBMTo9OnT0uS2rZtq2+//VaSZIzRmjVrFBAQoLVr10qSVq9erVtuuUU1atQoUE0PPfSQUlJS9PXXXysxMVFNmzZV+/btrfNL0o8//qhFixZpyZIlWrJkiVavXq033njD6h86dKjWrVunxYsXKz4+XmvWrNHWrVut/oULF6pKlSoaPXq0Tpw44RR4zp8/r4kTJ+qjjz5SQkKCjhw5on/+858FuhYA+UMgAlBslixZonLlysnb21sNGjRQSkqKhg0bJknKyMjQzJkzNWHCBHXq1El169bVe++9Jx8fH73//vuSLt9dWbt2rbKzs7Vz5055enoqNjbWCknffvut2rZtW6Da1q5dq++++04LFixQ8+bNVbNmTU2cOFEBAQH64osvrHE5OTmaM2eO6tevrzZt2qh3795asWKFpMt3h+bOnauJEyeqffv2ql+/vmbPnq3s7Gzr9YGBgSpTpozKly+v0NBQhYaGWn1ZWVmaNWuWmjdvrqZNmyouLs46NoCixRoiAMXmrrvu0syZM5WRkaHJkyfL3d1dPXr0kHT5zktWVpZatWpljffw8FCLFi20d+9eSVKbNm107tw5bdu2TevXr1fbtm3Vrl076w7N6tWrrYCVXzt27FB6eroqVqzo1P7bb7/pxx9/tParVaum8uXLW/uVK1dWSkqKJOmnn35SVlaWWrRoYfX7+/urVq1a11VD2bJlVb169aseG0DRIhABKDa+vr7W21kffPCBGjVqpPfff1/9+/e/rtcHBASoUaNG+vbbb7Vhwwbdc889uvPOO/XII49o//79OnDgQIHvEKWnp6ty5crW3aYrz5vLw8PDqc/hcCgnJ6dA57zS1Y5tjCmUYwP4Y7xlBsAl3Nzc9K9//UsvvviifvvtN1WvXl2enp5at26dNSYrK0ubN29W3bp1rba2bdtq1apVSkhIULt27RQYGKg6derotddeU+XKlXXbbbcVqJ6mTZsqKSlJ7u7uqlGjhtMWFBR0Xce49dZb5eHh4bQQ/OzZs9q/f7/TOE9PT6e30QC4HoEIgMs89NBDKlOmjGbMmCFfX1899dRTGjZsmJYtW6Y9e/boiSee0Pnz553uILVr107ffPON3N3dVbt2batt3rx513V3KDs7W9u3b3fa9u7dq+joaEVFRal79+5avny5fv75Z61fv14vvPCCtmzZcl3XU758efXp00fDhg3TqlWrtHv3bvXv319ubm5OT5NVq1ZNCQkJOnbsmE6ePJnPWQNQFAhEAFzG3d1dcXFxGj9+vDIyMvTGG2+oR48e6t27t5o2baqDBw/qm2++UYUKFazXtGnTRjk5OU7hp127dsrOzla7du3+9Jzp6elq0qSJ03bvvffK4XBo6dKluvPOO9WvXz/ddttt6tmzpw4fPqyQkJDrvqZJkyYpKipKXbt2VXR0tFq1aqU6derI29vbGjN69Gj9/PPPql69uipVqnTdxwZQdByGN6gBoMhkZGTolltu0Ztvvnnda6UAFD8WVQNAIdq2bZt++OEHtWjRQmfPntXo0aMlSd26dXNxZQD+CIEIAArZxIkTtW/fPnl6eqpZs2Zas2bNdS/MBuAavGUGAABsj0XVAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9ghEAADA9v4f+3rxbTrUI2wAAAAASUVORK5CYII=",
511 | "text/plain": [
512 | ""
513 | ]
514 | },
515 | "metadata": {},
516 | "output_type": "display_data"
517 | }
518 | ],
519 | "source": [
520 | "plot_histogram(df_filtered['row_length'])"
521 | ]
522 | },
523 | {
524 | "cell_type": "code",
525 | "execution_count": 9,
526 | "metadata": {},
527 | "outputs": [],
528 | "source": [
529 | "# drop un needed columns\n",
530 | "df_filtered = df_filtered.drop(['row_length'], axis=1)"
531 | ]
532 | },
533 | {
534 | "cell_type": "code",
535 | "execution_count": 10,
536 | "metadata": {},
537 | "outputs": [],
538 | "source": [
539 | "# convert to dataset and select num_samples (i.e. 1000) record for now (considering memory resource)\n",
540 | "dataset = Dataset.from_pandas(df_filtered)\n",
541 | "\n",
542 | "if num_samples: \n",
543 | " dataset = dataset.select(range(num_samples))"
544 | ]
545 | },
546 | {
547 | "cell_type": "code",
548 | "execution_count": 11,
549 | "metadata": {},
550 | "outputs": [
551 | {
552 | "data": {
553 | "text/plain": [
554 | "Dataset({\n",
555 | " features: ['text', '__index_level_0__'],\n",
556 | " num_rows: 1000\n",
557 | "})"
558 | ]
559 | },
560 | "execution_count": 11,
561 | "metadata": {},
562 | "output_type": "execute_result"
563 | }
564 | ],
565 | "source": [
566 | "dataset"
567 | ]
568 | },
569 | {
570 | "cell_type": "code",
571 | "execution_count": null,
572 | "metadata": {},
573 | "outputs": [],
574 | "source": []
575 | }
576 | ],
577 | "metadata": {
578 | "kernelspec": {
579 | "display_name": "Python 3",
580 | "language": "python",
581 | "name": "python3"
582 | },
583 | "language_info": {
584 | "codemirror_mode": {
585 | "name": "ipython",
586 | "version": 3
587 | },
588 | "file_extension": ".py",
589 | "mimetype": "text/x-python",
590 | "name": "python",
591 | "nbconvert_exporter": "python",
592 | "pygments_lexer": "ipython3",
593 | "version": "3.11.3"
594 | }
595 | },
596 | "nbformat": 4,
597 | "nbformat_minor": 2
598 | }
599 |
--------------------------------------------------------------------------------