Build a Data Connector

Quick Start

Code Structure

Clone the following repo: https://github.com/CliqueOfficial/clique-sibyl-demo-data-connector.

Assume the Data Connector is named SibylDemoDataConnector, the file structure should look like the following：

├── Dockerfile.sibyl
├── Dockerfile.DCsv2.sibyl
├── Dockerfile.DCsv2.pccs
├── docker-compose.yml
├── docker-compose-dcap.yml
├── connectors
│   └── SibylDemoDataConnector
│       ├── .cargo
│       │   └── config.toml
│       ├── Cargo.toml
│       └── src
│           ├── demo.rs
│           └── lib.rs
└── sibyl.toml

• Dockerfile.sibyl : build the sibyl image:

FROM asia-northeast1-docker.pkg.dev/clique-demo/clique-sibyl-base:2.2.2 AS BUILDER
COPY . $APP_DIR/SibylDataConnectors
RUN bash -c "$BUILD_SCRIPT"
RUN rm -rf $APP_DIR/SibylDataConnectors


FROM asia-northeast1-docker.pkg.dev/clique-demo/clique-sibyl-base:2.2.2
COPY --from=BUILDER $APP_DIR $APP_DIR

• Dockerfile.DCsv2.sibyl : build the DCsv2 sibyl image:

FROM asia-northeast1-docker.pkg.dev/clique-demo/clique-sibyl-dcsv2-base:2.2.2 AS BUILDER
COPY . $APP_DIR/SibylDataConnectors
RUN bash -c "$BUILD_SCRIPT"
RUN rm -rf $APP_DIR/SibylDataConnectors


FROM asia-northeast1-docker.pkg.dev/clique-demo/clique-sibyl-dcsv2-base:2.2.2
COPY --from=BUILDER $APP_DIR $APP_DIR

• Dockerfile.DCsv2.pccs : build the custom DCAP Service for DCsv2 :

FROM baiduxlab/sgx-rust@sha256:fbf4b495a0433ee2ef45ae9780b05d2f181aa6bbbe16dd0cf9ab5b4059ff15a5
RUN bash -c "wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add -"
RUN apt update && apt install -y libsgx-dcap-quote-verify=1.14.100.3-bionic1 libsgx-dcap-quote-verify-dev=1.14.100.3-bionic1 libsgx-dcap-default-qpl=1.14.100.3-bionic1 libsgx-dcap-default-qpl-dev=1.14.100.3-bionic1 libsgx-dcap-ql=1.14.100.3-bionic1 libsgx-dcap-ql-dev=1.14.100.3-bionic1 sgx-pck-id-retrieval-tool=1.14.100.3-bionic1
RUN cd /tmp && curl -o setup.sh -sL https://deb.nodesource.com/setup_14.x && chmod a+x setup.sh && ./setup.sh
RUN apt-get -y install nodejs sqlite3 python build-essential cracklib-runtime dkms
RUN mkdir /etc/init
RUN bash -c 'apt install -y sgx-dcap-pccs=1.14.100.3-bionic1 < <(printf "Y\n\n\nY\n8081\nN\ne0dd86294fb54b109e4ce6961af4b952\nLAZY\n97294597\n97294597\n97294597\n97294597\nY\nUS\nNY\nNY\nClique2046\n\n\[email protected]\n97294597\n\n")'
WORKDIR /opt/intel/sgx-dcap-pccs
ENTRYPOINT ["node", "-r", "esm", "pccs_server.js"]

• docker-compose.yml: deploy sibyl :

version: "3.1"
services:
  sibyl:
    image: sibyl
    devices:
      - /dev/sgx/enclave
      - /dev/sgx/provision
    restart: always
    ports:
      - 3443:3443
    command:
      - "--addr"
      - "0.0.0.0:3443"
      - "--maxconn"
      - "128"

• docker-compose-dcap.yml: deploy sibyl with custom DCAP service:

version: "3.1"
services:
  clique-pccs:
    image: pccs
    hostname: clique-pccs
    ports:
      - 8081:8081
    devices:
      - /dev/sgx/enclave
      - /dev/sgx/provision
    networks:
      - resolute
    restart: always

  sibyl:
    image: sibyl
    depends_on:
      - clique-pccs
    devices:
      - /dev/sgx/enclave
      - /dev/sgx/provision
    networks:
      - resolute
    restart: always
    ports:
      - 3443:3443
    command:
      - "--addr"
      - "0.0.0.0:3443"
      - "--maxconn"
      - "128"

networks:
  resolute:
    external:
      name: resolute

• connectors: source codes of the Data Connectors.

• sibyl.toml: sibyl config file.

Implement A Data Connector

• Generate connectors/SibylDemoDataConnector/.cargo/config.toml:

[net]
git-fetch-with-cli = true

[build]
rustflags = ["-Lprebuild"]

• Generate connectors/SibylDemoDataConnector/Cargo.toml:

[package]
name = "sibyl_demo_data_connector"
version = "2.1.0"
edition = "2018"
license = "MIT"
readme = "README.md"
include = ["src/**/*.rs", "README.md"]

[dependencies]

[features]
default = ["mesalock_sgx"]
mesalock_sgx = []

• Generate connectors/SibylDemoDataConnector/src/lib.rs:

#![cfg_attr(not(target_env = "sgx"), no_std)]

#[cfg(all(feature = "mesalock_sgx", not(target_env = "sgx")))]
#[macro_use]
extern crate sgx_tstd as std;
extern crate sibyl_base_data_connector;

pub mod demo;

• Generate connectors/SibylDemoDataConnector/src/demo.rs:

use std::prelude::v1::*;
use sibyl_base_data_connector::utils::simple_tls_client;
use sibyl_base_data_connector::base::DataConnector;
use sibyl_base_data_connector::serde_json::Value;
use sibyl_base_data_connector::errors::NetworkError;
use std::convert::TryInto;

pub struct DemoConnector {}

impl DataConnector for DemoConnector {
  fn query(&self, query_type: &Value, query_param: &Value) -> Result<Value, NetworkError> {
    let query_type_str = match query_type.as_str() {
      Some(r) => r,
      _ => {
        let err = format!("query_type to str failed");
        println!("{:?}", err);
        return Err(NetworkError::String(err));
      }
    };
    match query_type_str {
      "demo_post" => {
        let host = query_param["host"].as_str().unwrap_or("");
        let port = query_param["port"].as_i64().unwrap_or(443);
        let post_body = query_param["post_body"].as_str().unwrap_or("");
        let url = query_param["url"].as_str().unwrap_or("/");
        let req = format!(
          "POST {} HTTP/1.1\r\n\
          HOST: {}\r\n\
          User-Agent: curl/7.79.1\r\n\
          Accept: */*\r\n\
          Content-Type: application/json\r\n\
          Content-Length: {}\r\n\r\n\
          {}",
          url, host, post_body.len(), post_body 
        );
        simple_tls_client(host, &req, port.try_into().unwrap())
      },
      "demo_get" => {
        let host = query_param["host"].as_str().unwrap_or("");
        let port = query_param["port"].as_i64().unwrap_or(443);
        let url = query_param["url"].as_str().unwrap_or("/");
        let req = format!(
          "GET {} HTTP/1.1\r\n\
          HOST: {}\r\n\
          User-Agent: curl/7.79.1\r\n\
          Accept: */*\r\n\r\n", 
          url, host 
        );
        simple_tls_client(host, &req, port.try_into().unwrap())
      },
      _ => {
        Err(NetworkError::String(format!("Unexpected query_type: {:?}", query_type)))
      }
    }
  }
}

Sibyl Config

We need to generate sibyl.toml to configurate the data connector:

[connectors]
# The format of the connector name should be sibyl_{CONNECTOR_ROUTE}_data_connector
sibyl_demo_data_connector = { path = "./connectors/SibylDemoDataConnector" }

We will integrate sibyl_demo_data_connector into the Sibyl Service from the specified path.

Route

In connectors/SibylDemoDataConnector/src/demo.rs, query_type_str must start with demo.You can configure it through route :

[connectors]
sibyl_demo_data_connector = { route = 'demo_query', path = "./connectors/SibylDemoDataConnector" }

Now we will load the connector code from sibyl_demo_data_connector::demo_query::DemoConnector, and you should:

• Rename connectors/SibylDemoDataConnector/src/demo.rs to connectors/SibylDemoDataConnector/src/demo_query.rs.

• Replace demo_post and demo_get with demo_query_post and demo_query_get in connectors/SibylDemoDataConnector/src/demo_query.rs.

• Replace pub mod demo; with pub mod demo_query; in connectors/SibylDemoDataConnector/src/lib.rs.

Connector

By default, we will load the connector code from sibyl_demo_data_connector::demo::DemoConnector . You can configure it through connector:

[connectors]
sibyl_demo_data_connector = { connector = 'DemoQueryConnector', path = "./connectors/SibylDemoDataConnector" }

Now you can replace DemoConnector with DemoQueryConnector in connectors/SibylDemoDataConnector/src/demo.rs, and load the connector code from sibyl_demo_data_connector::demo::DemoQueryConnector.

Build & Deploy

• Build Sibyl:

docker build -t sibyl -f Dockerfile.sibyl .

• Build DCsv2 Sibyl:

docker build -t sibyl -f Dockerfile.DCsv2.sibyl .

• Build DCsv2 with custom DCAP service:

docker build -t pccs -f Dockerfile.DCsv2.pccs .

• Deploy Sibyl:

docker compose -f docker-compose.yml up

• Deploy Sibyl with custom DCAP service:

docker compose -f docker-compose-dcap.yml up

Then Sibyl will run and listen on port 3443.

For Azure VMs, custom DCAP service is only avaiable for DCsv2 and is not supported in DCsv3.

Test

For example, if you want to make a get query for https://api.github.com/repos/github/repo-name:

curl -k --location 'https://localhost:3443/query' \
--header 'Content-Type: application/json' \
--data '{
    "query_type": "demo_get",
    "query_param": {
        "host": "api.github.com",
        "url": "/repos/github/repo-name",
        "port": 443
    }
}'