Updates to entire rest gateway, fixing and optimizing.

README.md

@@ -2,25 +2,25 @@
 
 One Platform for REST, SOAP, GraphQL, gRPC and Websocket APIs. Fully managed with its own set of RESTful APIs. This is your APIs gateway to the world!
 
- [pygate.org](https://pygate.org)
+🔗 [pygate.org](https://pygate.org)
 
-## Roadmap
+No onboarding and no specialized Go or C expertise required. Just a simple, cost-effective API Gateway built in Python. Keep it simple, scalable, and efficient while giving developers everything they need to manage APIs with ease. 🐍
 
-- [x] Create proof of concept.
-- [ ] REST gateway implementation (in progress).
-- [ ] Code cleanup and testing.
+## MVP Roadmap 🚀
+- [ ] REST gateway implementation (in progress ⏳).
+- [ ] Code optimization and testing.
 - [ ] Add REST capabilties to user documentation.
 - [ ] Version 1.0.0 release.
 - [ ] GraphQL gateway implementation.
-- [ ] Code cleanup and testing.
+- [ ] Code optimization and testing.
 - [ ] Add GraphQL capabilties to user documentation.
 - [ ] Version 1.1.0 release.
 - [ ] gRPC gateway implementation.
-- [ ] Code cleanup and testing.
+- [ ] Code optimization and testing.
 - [ ] Add gRPC capabilties to user documentation.
 - [ ] Version 1.2.0 release.
 - [ ] Websockets gateway implementation.
-- [ ] Code cleanup and testing.
+- [ ] Code optimization and testing.
 - [ ] Add Websockets capabilties to user documentation.
 - [ ] Version 1.3.0 release.
 - [ ] Improve caching.
@@ -54,6 +54,12 @@ Stop pygate
   python pygate.py stop
 ```
 
+Run pygate in console
+    
+```bash
+  python pygate.py run
+```
+
 
 
 ## License Information

models/api_model.py

@@ -9,13 +9,13 @@ from typing import List, Optional
 
 class ApiModel(BaseModel): 
     
-    api_id:  str
     api_name: str = Field(..., min_length=1, max_length=25)
     api_version: str = Field(..., min_length=1, max_length=2)
+    api_description: str = Field(None, min_length=1, max_length=127)
+    api_servers: List[str] = Field(default_factory=list)
+    api_type: str = None
     
-    api_description: Optional[str] = Field(None, min_length=1, max_length=127)
-    api_servers: Optional[List[str]] = Field(default_factory=list)
-    api_type: Optional[str] = None
+    api_id: Optional[str] = None
     api_path: Optional[str] = None
 
     class Config:

models/endpoint_model.py

@@ -13,7 +13,10 @@ class EndpointModel(BaseModel):
     api_version: str = Field(..., min_length=1, max_length=10)
     endpoint_method: str = Field(...) 
     endpoint_uri: str = Field(..., min_length=1, max_length=255)
-    description: Optional[str] = Field(..., min_length=1, max_length=255)
+
+    api_id: Optional[str] = Field(None, min_length=1, max_length=255)
+    description: Optional[str] = Field(None, min_length=1, max_length=255)
+    endpoint_id: Optional[str] = Field(None, min_length=1, max_length=255)
 
     class Config:
         arbitrary_types_allowed = True

models/request_model.py

@@ -1,15 +1,10 @@
-"""
-The contents of this file are property of pygate.org
-Review the Apache License 2.0 for valid authorization of use
-See https://github.com/pypeople-dev/pygate for more information
-"""
+from pydantic import BaseModel
+from typing import Dict, Optional
 
-class RequestModel:
-
-    def __init__ (self, method = None, path = None, headers = None, json = None, args = None, user = None):
-        self.method = method
-        self.path = path
-        self.headers = headers
-        self.json = json
-        self.args = args
-        self.user = user
+class RequestModel(BaseModel):
+    method: str
+    path: str
+    headers: Dict[str, str]
+    query_params: Dict[str, str]
+    identity: Optional[str] = None
+    body: Optional[str] = None

pygate.py

@@ -4,7 +4,7 @@ Review the Apache License 2.0 for valid authorization of use
 See https://github.com/pypeople-dev/pygate for more information
 """
 
-from fastapi import FastAPI, Request
+from fastapi import FastAPI
 from fastapi.responses import JSONResponse
 from fastapi_jwt_auth import AuthJWT
 from fastapi_jwt_auth.exceptions import AuthJWTException
@@ -38,7 +38,6 @@ load_dotenv()
 PID_FILE = "pygate.pid"
 
 pygate = FastAPI()
-
 origins = os.getenv("ALLOWED_ORIGINS", "http://localhost:3000").split(",")
 credentials = os.getenv("ALLOW_CREDENTIALS", "true").lower() == "true"
 methods = os.getenv("ALLOW_METHODS", "GET, POST, PUT, DELETE").split(",")
@@ -50,8 +49,8 @@ pygate.add_middleware(
     CORSMiddleware,
     allow_origins=origins,
     allow_credentials=credentials,
-    allow_methods=[methods],
-    allow_headers=[headers],
+    allow_methods=methods,
+    allow_headers=headers,
 )
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

routes/api_routes.py

@@ -31,11 +31,11 @@ Response:
 }
 """
 @api_router.post("")
-@auth_required()
-@whitelist_check()
-@role_required(("admin", "dev", "platform"))
 async def create_api(api_data: ApiModel):
     try:
+        auth_required()
+        whitelist_check()
+        role_required(("admin", "dev", "platform"))
         await ApiService.create_api(api_data)
         return JSONResponse(content={'message': 'API created successfully'}, status_code=201)
     except ValueError as e:

routes/authorization_routes.py

@@ -43,7 +43,7 @@ async def login(request: Request, Authorize: AuthJWT = Depends()):
     try:
         user = await UserService.check_password_return_user(email, password)
         access_token = create_access_token({"sub": user["username"], "role": user["role"]}, Authorize)
-        response = JSONResponse(content={"message": "You are logged in"}, media_type="application/json")
+        response = JSONResponse(content={"access_token": access_token}, media_type="application/json")
         Authorize.set_access_cookies(access_token, response)
         return response
     except ValueError as e:
@@ -62,12 +62,14 @@ Response:
 }
 """
 @authorization_router.get("/authorization/status")
-@auth_required()
-async def status():
+async def status(Authorize: AuthJWT = Depends()):
     try:
+        auth_required()
         return JSONResponse(content={"status": "authorized"}, status_code=200)
     except Exception as e:
         raise HTTPException(status_code=401, detail="Invalid token")
+    except ValueError as e:
+        return JSONResponse(content={"error": str(e)}, status_code=400)
         
 """
 Logout endpoint
@@ -80,17 +82,22 @@ Response:
 }
 """
 @authorization_router.post("/authorization/invalidate")
-@auth_required()
 async def logout(response: Response, Authorize: AuthJWT = Depends()):
     try:
+        auth_required()
         jwt_id = Authorize.get_raw_jwt()['jti']
         user = Authorize.get_jwt_subject()
         Authorize.unset_jwt_cookies(response)
-        # Add JWT ID to blacklist
         if user not in jwt_blacklist:
             jwt_blacklist[user] = TimedHeap()
         jwt_blacklist[user].push(jwt_id)
         return JSONResponse(content={"message": "Your token has been invalidated"}, status_code=200)
     except AuthJWTException as e:
         logging.error(f"Logout failed: {str(e)}")
-        return JSONResponse(status_code=500, content={"detail": "An error occurred during logout"})
+        return JSONResponse(status_code=500, content={"detail": "An error occurred during logout"})
+    except ValueError as e:
+        return JSONResponse(content={"error": str(e)}, status_code=400)
+    
+@authorization_router.api_route("/status", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"])
+async def rest_gateway():
+    return JSONResponse(content={"message": "Gateway is online"}, status_code=200)

routes/endpoint_routes.py

@@ -31,12 +31,12 @@ Response:
 }
 """
 @endpoint_router.post("")
-@auth_required()
-@whitelist_check()
-@role_required(("admin", "dev", "platform"))
 async def create_endpoint(endpoint_data: EndpointModel, Authorize: AuthJWT = Depends()):
     try:
-        EndpointService.create_endpoint(endpoint_data)
+        auth_required()
+        whitelist_check()
+        role_required(("admin", "dev", "platform"))
+        await EndpointService.create_endpoint(endpoint_data)
         return JSONResponse(content={'message': 'Endpoint created successfully'}, status_code=201)
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))

routes/gateway_routes.py

@@ -17,17 +17,21 @@ from models.request_model import RequestModel
 gateway_router = APIRouter()
 
 @gateway_router.api_route("/rest/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"])
-@auth_required()
-@whitelist_check()
-@subscription_required()
 async def rest_gateway(path: str, request: Request, Authorize: AuthJWT = Depends()):
-    request_model = RequestModel(
-        method=request.method,
-        path=path,
-        headers=dict(request.headers),
-        body=await request.json() if request.method in ["POST", "PUT", "PATCH"] else None,
-        query_params=dict(request.query_params),
-        identity=Authorize.get_jwt_subject()
-    )
-    response = GatewayService.rest_gateway(request_model)
-    return JSONResponse(content=response)
+    try:
+        auth_required()
+        whitelist_check()
+        subscription_required()
+
+        request_model = RequestModel(
+            method=request.method,
+            path=path,
+            headers=dict(request.headers),
+            body=await request.json() if request.method in ["POST", "PUT", "PATCH"] else None,
+            query_params=dict(request.query_params),
+            identity=Authorize.get_jwt_subject()
+        )
+
+        return await GatewayService.rest_gateway(request_model)
+    except ValueError as e:
+        return JSONResponse(content={"error": str(e)}, status_code=400)

routes/user_routes.py

@@ -39,13 +39,13 @@ Response:
     }
 }
 """
-@user_router.post("/")
-@auth_required()
-@role_required(["admin", "dev", "platform"])
-async def create_user(user_data: UserModel):
+@user_router.post("")
+async def create_user(user_data: UserModel, Authorize: AuthJWT = Depends()):
     try:
+        auth_required()
+        role_required(["admin", "dev", "platform"])
         new_user = await UserService.create_user(user_data)
-        return JSONResponse(content={"message": "User created successfully", "user_details": new_user}, status_code=201)
+        return JSONResponse(content={"message": "User created successfully"}, status_code=201)
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
 

services/api_service.py

@@ -19,12 +19,22 @@ class ApiService:
         """
         Onboard an API to the platform.
         """
-        if pygate_cache.get_cache('api_cache', f"{data.api_name}/{data.api_version}") or ApiService.api_collection.find_one({'api_name': data.api_name, 'api_version': data.api_version}):
+        cache_key = f"{data.api_name}/{data.api_version}"
+
+        if pygate_cache.get_cache('api_cache', cache_key) or ApiService.api_collection.find_one({'api_name': data.api_name, 'api_version': data.api_version}):
             raise ValueError("API already exists for the requested name and version")
-        data.api_path = f"/{data.get('api_name')}/{data.get('api_version')}"
+
+        data.api_path = f"/{data.api_name}/{data.api_version}"
         data.api_id = str(uuid.uuid4())
-        api = ApiService.api_collection.insert_one(data)
-        pygate_cache.set_cache('api_cache', f"{data.api_name}/{data.api_version}", api)
+
+        api_dict = data.dict()
+        insert_result = ApiService.api_collection.insert_one(api_dict)
+
+        if not insert_result.acknowledged:
+            raise ValueError("Database error: Unable to insert endpoint")
+        
+        api_dict['_id'] = str(insert_result.inserted_id)
+        pygate_cache.set_cache('api_cache', data.api_id, api_dict)
         pygate_cache.set_cache('api_id_cache', data.api_path, data.api_id)
 
     @staticmethod

services/cache.py

@@ -26,7 +26,8 @@ class PygateCacheManager:
             'user_cache': 'user_cache:',
             'user_group_cache': 'user_group_cache:',
             'user_role_cache': 'user_role_cache:',
-            'endpoint_load_balancer': 'endpoint_load_balancer:'
+            'endpoint_load_balancer': 'endpoint_load_balancer:',
+            'endpoint_server_cache': 'endpoint_server_cache:'
         }
 
     def _get_key(self, cache_name, key):

services/endpoint_service.py

@@ -13,22 +13,44 @@ import uuid
 
 class EndpointService:
     endpoint_collection = db.endpoints
+    apis_collection = db.apis
 
     @staticmethod
     async def create_endpoint(data: EndpointModel):
         """
         Create an endpoint for an API.
         """
-        if pygate_cache.get_cache('endpoint_cache', f"{data.api_name}/{data.api_version}/{data.endpoint_uri}") or EndpointService.endpoint_collection.find_one({
+        cache_key = f"/{data.api_name}/{data.api_version}/{data.endpoint_uri}".replace("//", "/")
+
+        if pygate_cache.get_cache('endpoint_cache', cache_key) or EndpointService.endpoint_collection.find_one({
             'api_name': data.api_name,
             'api_version': data.api_version,
             'endpoint_uri': data.endpoint_uri
         }):
             raise ValueError("Endpoint already exists for the requested API")
-        
-        data['endpoint_id'] = str(uuid.uuid4())
-        endpoint = EndpointService.endpoint_collection.insert_one(data)
-        pygate_cache.set_cache('endpoint_cache', f"{data.getapi_name}/{data.api_version}/{data.endpoint_uri}", endpoint)
+
+        data.api_id = pygate_cache.get_cache('api_id_cache', data.api_name + '/' + data.api_version)
+
+        if not data.api_id:
+            api = EndpointService.apis_collection.find_one({"api_name": data.api_name, "api_version": data.api_version})
+            if not api:
+                raise ValueError("API does not exist")
+            data.api_id = api.get('api_id')
+            pygate_cache.set_cache('api_id_cache', f"{data.api_name}/{data.api_version}", data.api_id)
+
+        data.endpoint_id = str(uuid.uuid4())
+
+        endpoint_dict = data.dict()
+        insert_result = EndpointService.endpoint_collection.insert_one(endpoint_dict)
+
+        if not insert_result.acknowledged:
+            raise ValueError("Database error: Unable to insert endpoint")
+
+        endpoint_dict['_id'] = str(insert_result.inserted_id)
+        pygate_cache.set_cache('endpoint_cache', cache_key, endpoint_dict)
+        api_endpoints = pygate_cache.get_cache('api_endpoint_cache', data.api_id) or list()
+        api_endpoints.append(endpoint_dict.get('endpoint_method') + endpoint_dict.get('endpoint_uri'))
+        pygate_cache.set_cache('api_endpoint_cache', data.api_id, api_endpoints)
 
     @staticmethod
     @cache_manager.cached(ttl=300)

services/gateway_service.py

@@ -4,12 +4,19 @@ Review the Apache License 2.0 for valid authorization of use
 See https://github.com/pypeople-dev/pygate for more information
 """
 
+import re
+import time
+from fastapi.responses import JSONResponse
 import requests
+import logging
 
 from utils.database import db
 from services.cache import pygate_cache
 
 class GatewayService:
+    logging.basicConfig(level=logging.INFO, format='%(levelname)s - %(message)s')
+    logger = logging.getLogger("pygate.gateway")
+    
     api_collection = db.apis
     endpoint_collection = db.endpoints
 
@@ -18,23 +25,53 @@ class GatewayService:
         """
         External gateway.
         """
-        api = pygate_cache.get_cache('api_id_cache', request.path) or GatewayService.api_collection.find_one({'api_path': request.path})
-        if not api:
-            raise ValueError("API does not exists")
-        endpoints = pygate_cache.get_cache('api_endpoint_cache', api.get('api_id')) or GatewayService.endpoint_collection.find({'api_id': api.get('api_id')})
-        if not endpoints or request.path not in endpoints:
-            raise ValueError("Endpoint does not exists")
-        server_index = pygate_cache.get_cache('endpoint_server_cache', api.get('api_id')) or 0
-        server = api.get('api_servers')[server_index]
-        pygate_cache.set_cache('endpoint_server_cache', api.get('api_id'), (server_index + 1) % len(api.get('api_servers')))
-        url = server + request.path
-        response = None
-        if request.method == 'GET':
-            response = requests.get(url)
-        elif request.method == 'POST':
-            response = requests.post(url, json=request.json)
-        elif request.method == 'PUT':
-            response = requests.put(url, json=request.json)
-        elif request.method == 'DELETE':
-            response = requests.delete(url)
-        return response
+        start_time = time.time() * 1000
+        gateway_end_time = None
+        backend_start_time = None
+        try:
+            match = re.match(r"([^/]+/v\d+)", request.path)
+            api_name_version = '/' + match.group(1) if match else ""
+            endpoint_uri = re.sub(r"^[^/]+/v\d+/", "", request.path)
+            api = pygate_cache.get_cache('api_cache', pygate_cache.get_cache('api_id_cache', api_name_version)) or GatewayService.api_collection.find_one({'api_path': api_name_version})   
+            if not api:
+                raise ValueError("API does not exists: " + api_name_version)
+            endpoints = pygate_cache.get_cache('api_endpoint_cache', api.get('api_id')) or GatewayService.endpoint_collection.find({'api_id': api.get('api_id')})
+            if not endpoints or not any(re.fullmatch(re.sub(r"\{[^/]+\}", r"([^/]+)", endpoint), request.method + '/' + endpoint_uri) for endpoint in endpoints):
+                raise ValueError("Endpoint does not exists - " + str(endpoints) + "-" + request.method + '/' + endpoint_uri)
+            server_index = pygate_cache.get_cache('endpoint_server_cache', api.get('api_id')) or 0
+            api_servers = api.get('api_servers') or []
+            server = api_servers[server_index]
+            pygate_cache.set_cache('endpoint_server_cache', api.get('api_id'), (server_index + 1) % len(api.get('api_servers')))
+            url = server + request.path
+            gateway_end_time = time.time() * 1000
+            backend_start_time = time.time() * 1000
+            method = request.method.upper()
+            if method == 'GET':
+                response = requests.get(url)
+            elif method == 'POST':
+                body = await request.json()
+                response = requests.post(url, json=body)
+            elif method == 'PUT':
+                body = await request.json()
+                response = requests.put(url, json=body)
+            elif method == 'DELETE':
+                response = requests.delete(url)
+            else:
+                return JSONResponse(content={"error": "Method not supported"}, status_code=405)
+            try:
+                response_content = response.json()
+            except requests.exceptions.JSONDecodeError:
+                response_content = response.text
+            if response.status_code == 404:
+                return JSONResponse("Endpoint does not exists in backend service", status_code=404)
+            return JSONResponse(content=response_content, status_code=response.status_code)
+        except Exception as e:
+            GatewayService.logger.error(f"Error in rest_gateway: {str(e)}")
+            return {"error": str(e)}
+        finally:
+            end_time = time.time() * 1000
+            if gateway_end_time:
+                GatewayService.logger.info(f"Gateway Time: {gateway_end_time - start_time}ms")
+            if backend_start_time:
+                GatewayService.logger.info(f"Backend Time: {end_time - backend_start_time}ms")
+            GatewayService.logger.info(f"Total Time: {end_time - start_time}ms")

services/user_service.py

@@ -55,15 +55,12 @@ class UserService:
         if UserService.user_collection.find_one({'email': data.email}):
             raise ValueError("Email already exists")
 
-        data['password'] = password_util.hash_password(data.password)
-        
-        user = UserService.user_collection.insert_one(data)
-        pygate_cache.get_cache('user_cache', data.username, user)
-        
-        return {
-            'username': data.username,
-            'email': data.email
-        }
+        data.password = password_util.hash_password(data.password)
+
+        data_dict = data.dict()
+        user = UserService.user_collection.insert_one(data_dict)
+        data_dict['_id'] = str(user.inserted_id)
+        pygate_cache.set_cache('user_cache', data.username, data_dict)
 
     @staticmethod
     async def check_password_return_user(email, password):

tests/rest_gateway_tests.py

@@ -0,0 +1,103 @@
+import json
+import random
+import unittest
+import time
+import requests
+
+class TestPygate(unittest.TestCase):
+    base_url = "http://localhost:3002"
+    token = None
+    api_name = None
+    endpoint_path = None
+
+    @classmethod
+    def setUpClass(cls):
+        for _ in range(5):
+            try:
+                response = requests.get(f"{cls.base_url}/platform/status")
+                if response.status_code == 200:
+                    print("Server started successfully")
+                    break
+            except requests.exceptions.ConnectionError:
+                print("Failed to connect to the server, retrying...")
+                time.sleep(2)
+        else:
+            print("Failed to connect to the server after multiple attempts")
+            raise RuntimeError("pygate is not running")
+
+    def test_01_auth_calls(self):
+        response = requests.post(f"{self.base_url}/platform/authorization", 
+                                 json={"email": "admin@pygate.org", "password": "password1"})
+        self.assertEqual(response.status_code, 200)
+
+        TestPygate.token = response.json().get('access_token') 
+        self.assertIsNotNone(TestPygate.token)
+
+        response = requests.get(f"{self.base_url}/platform/authorization/status",
+                                headers={"Authorization": f"Bearer {TestPygate.token}"})
+        self.assertEqual(response.status_code, 200)
+
+    def test_02_create_user(self):
+        if not TestPygate.token:
+            self.fail("Auth token is missing")
+
+        response = requests.post(f"{self.base_url}/platform/user", 
+                                 headers={"Authorization": f"Bearer {TestPygate.token}"},
+                                 json={
+                                     "username": "newuser" + str(time.time()), 
+                                     "email": "newuser" + str(time.time()) + "@pygate.org", 
+                                     "password": "newpass", 
+                                     "role": "user"
+                                 })
+        self.assertEqual(response.status_code, 201)
+
+    def test_03_onboard_api(self):
+        """Step 3: Onboard an API"""
+        if not TestPygate.token:
+            self.fail("Auth token is missing")
+
+        TestPygate.api_name = "test" + "".join(random.sample("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ", 8))
+        
+        response = requests.post(f"{self.base_url}/platform/api", 
+                                 headers={"Authorization": f"Bearer {TestPygate.token}"},
+                                 json={
+                                     "api_name": TestPygate.api_name,
+                                     "api_version": "v1", 
+                                     "api_description": "Test API", 
+                                     "api_servers": ["https://fake-json-api.mock.beeceptor.com/"], 
+                                     "api_type": "REST"
+                                 })
+        self.assertEqual(response.status_code, 201)
+
+    def test_04_onboard_endpoint(self):
+        if not TestPygate.token:
+            self.fail("Auth token is missing")
+
+        TestPygate.endpoint_path = "/users"
+
+        response = requests.post(f"{self.base_url}/platform/endpoint", 
+                                 headers={"Authorization": f"Bearer {TestPygate.token}"},
+                                 json={
+                                     "api_name": TestPygate.api_name,
+                                     "api_version": "v1", 
+                                     "endpoint_uri": TestPygate.endpoint_path,
+                                     "endpoint_method": "GET"
+                                 })
+        self.assertEqual(response.status_code, 201)
+
+    def test_05_gateway_call(self):
+        response = requests.get(f"{self.base_url}/api/rest/" + TestPygate.api_name + "/v1" + TestPygate.endpoint_path.replace("{userId}", "2"))
+        self.assertEqual(response.status_code, 200)
+
+def suite():
+    test_suite = unittest.TestSuite()
+    test_suite.addTest(TestPygate("test_01_auth_calls"))
+    test_suite.addTest(TestPygate("test_02_create_user"))
+    test_suite.addTest(TestPygate("test_03_onboard_api"))
+    test_suite.addTest(TestPygate("test_04_onboard_endpoint"))
+    test_suite.addTest(TestPygate("test_05_gateway_call"))
+    return test_suite
+
+if __name__ == '__main__':
+    runner = unittest.TextTestRunner()
+    runner.run(suite())

utils/database.py

@@ -33,7 +33,7 @@ class Database:
 
         self.db.endpoints.create_indexes([
             IndexModel([("api_id", ASCENDING)], unique=True),
-            IndexModel([("api_name", ASCENDING), ("version", ASCENDING)]),
+            IndexModel([("api_name", ASCENDING), ("version", ASCENDING)], unique=True),
             IndexModel([("api_name", ASCENDING), ("version", ASCENDING), ("path", ASCENDING)], unique=True)
         ])