Added infinite scrolling, Fixed brightdata proxy support, Updated search endpoint

README.md

@@ -1,3 +1,7 @@
+
+
+
+
 <img src="./docs/assets/osintbuddy.svg" height="80px" />
 
 ---
@@ -54,23 +58,33 @@ docker-compose up -d
 
 ## Progress Notes
 
-- Dockerized the Scrapy crawler and added to stack, made plans to add CSE link updating at regular intervals - [Sept 21, 2021]
-- 
+- [Sept 21, 2021]
+  - Dockerized the Scrapy crawler and added to stack, made plans to add CSE link updating at regular intervals
+- [Sept 28, 2021]
+  - Fixed bug in parsing the 'cse.js' response and massively increased crawler success rate (HTTP 200) when scraping CSE urls
+  -  Implemented proxy support that will rotate through a list of proxies
+  - Added support for a brightdata.com proxy list
+  - Added infinite scrolling to CSE results on the frontend
+  - Started adding endpoints for querying search results
 
 ### Progress Screenshots
 
+Added infinite scrolling to results
+
+<img alt="" src="./docs/assets/OB-infinite_scroll.gif" />
+
+#### Crawler Demo
+
+Frontend POSTs to backend, backend POSTs to crawler which spawns a Scrapy Spider for saving CSE results to the database (Much faster than pictured below, this gif is from before a bug was fixed)
+
 <img alt="" src="./docs/assets/osint_buddy_demo.gif" />
 
-
-
-
-
 #### Mobile Results
 
 <img alt="OsintBuddy" src="./docs/assets/OB_CSE-mobile-results.gif" align="center" />
 
 
-#### Latest Database Implementation
+#### Old Database Implementation
 
 <img alt="OsintBuddy" src="./docs/assets/OB-database.gif" />
 
@@ -91,22 +105,3 @@ docker-compose up -d
 
 <img alt="OsintBuddy" src="./docs/assets/OB-cse-crawler.gif" />
 
-
-#### Docs overview
-
-<img alt="OsintBuddy" src="./docs/assets/OB-docs-overview.gif" />
-
-
-#### Flower
-
-<img alt="OsintBuddy" src="./docs/assets/OB-flower.gif" />
-
-
-#### PGAdmin
-
-<img alt="OsintBuddy" src="./docs/assets/OB-pgadmin.gif" />
-
-
-#### Traefik
-
-<img alt="OsintBuddy" src="./docs/assets/OB-traefik.gif" />

backend/app/app/api/api_v1/api.py

@@ -6,5 +6,4 @@ api_router = APIRouter()
 api_router.include_router(login.router, tags=["login"])
 api_router.include_router(users.router, prefix="/users", tags=["users"])
 api_router.include_router(utils.router, prefix="/utils", tags=["utils"])
-api_router.include_router(items.router, prefix="/items", tags=["items"])
 api_router.include_router(search.router, prefix="/search", tags=["search"])

backend/app/app/api/api_v1/endpoints/search.py

@@ -44,12 +44,13 @@ def get_search(
         db: Session = Depends(deps.get_db),
         # current_user: models.User = Depends(deps.get_current_active_user),
         searchId: int,  # noqa
-        min: int,
-        max: int
+        limit: int,
+        offset: int
 ) -> Any:
     """
-    TODO:
+    TODO: Create Model for results response, create max result return limit
+     how much filtering should be done client side?
     """
-    data = crud.search_result.get_by_limit_offset(db, searchId, min, max)
-    print('data', data)
-    return {"hello": data}
+    data = crud.search_result.get_by_limit_offset(db, searchId, limit, offset)
+    total_results_count = crud.search_result.get_count(db, searchId)
+    return {"total_results": total_results_count, "results_count": len(data), "results": data}

backend/app/app/crud/crud_search_result.py

@@ -10,20 +10,13 @@ from app.schemas.search_result import SearchResultCreate, SearchResultUpdate, Se
 
 
 class CRUDSearchResult(CRUDBase[SearchResult, SearchResultCreate, SearchResultUpdate]):
-
-    def get_by_limit_offset(self, db: Session, search_id: int, min: int = 0, max: int = 100) -> Optional[SearchResult]:  # noqa
-        query = db.query(self.model).filter(self.model.search_id == id)
-        listen(query, 'before_compile', self._apply_limit(db, min, max), retval=True)
+    def get_by_limit_offset(self, db: Session, search_id: int, limit: int = 0, offset: int = 100) -> Optional[SearchResult]:  # noqa
+        query = db.query(self.model).with_entities(self.model.id, self.model.title, self.model.description, self.model.url).filter(self.model.search_id == search_id).limit(limit).offset(offset).all()
         return query
 
-    def _apply_limit(self, db: Session, min: int, max: int):  # noqa
-        def wrapped(query: db.query):
-            if max:
-                query = query.limit(max)
-                if min:
-                    query = query.offset(min * max)
-            return query
-        return wrapped
+    def get_count(self, db: Session, search_id: int):
+        query = db.query(self.model).filter(self.model.search_id == search_id).count()
+        return query
 
 
 search_result = CRUDSearchResult(Search_Result)

crawler/crawler/brightdata_proxies.py

@@ -39,7 +39,7 @@ def parse_proxies(proxy_file: str, is_brightdata_provider: bool = CUSTOMER_ID):
 
 
 class ProxyGenerator(object):
-    def __init__(self, is_brightdata_provider: Optional[bool] = False, proxy_file: str = '/spiderman/crawler/ips-cse_zone.txt'):
+    def __init__(self, is_brightdata_provider: Optional[bool] = CUSTOMER_ID, proxy_file: str = '/spiderman/crawler/ips-cse_zone.txt'):
         self.proxies = parse_proxies(proxy_file=proxy_file, is_brightdata_provider=True)
         self.is_brightdata_provider = is_brightdata_provider
 
@@ -57,7 +57,7 @@ class ProxyGenerator(object):
 
 def get_proxies():
     proxy_list = []
-    for p in ProxyGenerator(is_brightdata_provider=True):
+    for p in ProxyGenerator():
         proxy_list.append(p)
     return proxy_list
 

crawler/crawler/spiders/cse.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# All of this file needs some work, could be more intelligent and throw away less urls
 import os
 import re
 import json
@@ -28,7 +27,7 @@ class CseSpider(scrapy.Spider):
 
     custom_settings = dict()
     custom_settings['ITEM_PIPELINES'] = {'crawler.cse_pipeline.CsePipeline':  400}
-    custom_settings['DOWNLOADER_MIDDLEWARES']= {'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810}  # noqa
+    custom_settings['DOWNLOADER_MIDDLEWARES'] = {'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810}  # noqa
     custom_settings['BOT_NAME'] = 'CSE-Buddy'
     # CSEs by default have no robots.txt || Last checked September 16, 2021
     custom_settings['ROBOTSTXT_OBEY'] = False

crawler/service.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+#TODO: Implement this -> https://docs.scrapy.org/en/latest/topics/stats.html
 import datetime
 from queue import Queue
 
@@ -37,7 +38,7 @@ def start_crawl(search_in: CrawlRequestSchema):
     try:
         crawl_scheduler.loop_crawl(CseSpider, crawl_queue.qsize, **data)
     except Exception as e:
-        print("TODO...:", e)
+        print("TODO, more research needed: ", e)
     finally:
         data['created'] = datetime.datetime.now()
         return CrawlResponseSchema(**data)

frontend/package.json

@@ -41,6 +41,7 @@
     "sass": "^1.26.5",
     "sass-loader": "^8.0.2",
     "tailwindcss": "npm:@tailwindcss/postcss7-compat@^2.2.7",
+    "vue-infinite-loading": "^2.4.5",
     "vue-template-compiler": "^2.6.11"
   }
 }

frontend/src/api.js

@@ -16,5 +16,10 @@ export const api = {
     },
     async createSearch(token, data) {
         return axios.post(`${apiUrl}/api/v1/search/`, data, authHeaders(token))
+    },
+    async getSearchResults(token, searchId, limit, offset) {
+        // GET Endpoint: /api/v1/search/
+        // Example filter: localhost/api/v1/search/?searchId=5&limit=500&offset=6500
+        return axios.get(`${apiUrl}/api/v1/search/?searchId=${searchId}&limit=${limit}&offset=${offset}`, authHeaders(token))
     }
 }

frontend/src/components/cards/data/AnyResultCard.vue

@@ -1,5 +1,5 @@
 <template>
-  <section class="flex shadow-9 bg-white-100 relative">
+  <section class="flex shadow-9 bg-white-100 relative my-4">
     <section
         :class="
         isSaved
@@ -15,7 +15,7 @@
               <h2
                   class="text-black-500 leading-6 font-body underline font-medium"
               >
-                {{ title }}
+                {{ source.title }}
               </h2>
               <component :is="cloudIcon" class="text-primary-300 pr-2 opacity-40" />
               <!--              <h2 class="font-semibold font-body text-black-500"></h2>-->
@@ -28,7 +28,7 @@
                 <h2
                     class="text-black-300 leading-6 font-medium font-body text-sm"
                 >
-                  {{ description }}
+                  {{ source.description }}
                 </h2>
 
               </div>
@@ -36,9 +36,9 @@
             </section>
           </section>
           <section class="lg:visible lg:relative hidden invisible">
-            <div class="h-20 w-20 ">
-              <img alt="" class="w-full h-full" :src="imgUrl" />
-            </div>
+<!--            <div class="h-20 w-20 ">-->
+<!--              <img alt="" class="w-full h-full" :src="imgUrl" />-->
+<!--            </div>-->
           </section>
         </section>
 
@@ -46,12 +46,12 @@
           <section>
             <a
                 class="font-head flex"
-                :href="url"
+                :href="source.url"
             >
               <p
                   class="text-sm hover:underline text-primary-400  overflow-ellipsis whitespace-wrap max-w-lg leading-6 font-body "
               >
-                {{ domain }}
+                {{ source.url }}
               </p>
             </a>
           </section>
@@ -76,31 +76,16 @@ export default {
   components: {
     btn: SimpleBtn,
   },
-
   props: {
-
-    domain: {
-      default: null,
-      value: String,
-    },
-    url: {
-      default: null,
-      value: String,
-    },
-    title: {
-      default: null,
-      value: String,
-    },
-    description: {
-      default: null,
-      value: String,
-    },
-    imgUrl: {
-      default: null,
-      value: String,
+    source: {
+      type: Object,
+      default () {
+        return {}
+      }
     },
   },
 
+
   data() {
     return {
       isSaved: false,

frontend/src/components/forms/SearchBarAlternative.vue

@@ -35,7 +35,12 @@ export default {
 
   methods: {
     submitSearch() {
+
       api.createSearch(this.$store.getters.isLoggedIn, {query: this.searchInput})
+          .then(resp => {
+            // TODO: Update Store to store response
+            console.log(resp.data)
+          })
     }
   },
 

frontend/src/main.js

@@ -27,5 +27,5 @@ Vue.config.productionTip = false
 new Vue({
   router,
   store,
-  render: h => h(App)
+  render: h => h(App),
 }).$mount('#app')

frontend/yarn.lock

@@ -11117,6 +11117,11 @@ vue-hot-reload-api@^2.3.0:
   resolved "https://registry.yarnpkg.com/vue-hot-reload-api/-/vue-hot-reload-api-2.3.4.tgz#532955cc1eb208a3d990b3a9f9a70574657e08f2"
   integrity sha512-BXq3jwIagosjgNVae6tkHzzIk6a8MHFtzAdwhnV5VlvPTFxDCvIttgSiHWjdGoTJvXtmRu5HacExfdarRcFhog==
 
+vue-infinite-loading@^2.4.5:
+  version "2.4.5"
+  resolved "https://registry.yarnpkg.com/vue-infinite-loading/-/vue-infinite-loading-2.4.5.tgz#cc20fd40af7f20188006443c99b60470cf1de1b3"
+  integrity sha512-xhq95Mxun060bRnsOoLE2Be6BR7jYwuC89kDe18+GmCLVrRA/dU0jrGb12Xu6NjmKs+iTW0AA6saSEmEW4cR7g==
+
 vue-jest@^3.0.5:
   version "3.0.7"
   resolved "https://registry.yarnpkg.com/vue-jest/-/vue-jest-3.0.7.tgz#a6d29758a5cb4d750f5d1242212be39be4296a33"