셰이드 파인더(Shade Finder) 앱 개발: 트웰브랩스(Twelve Labs) API를 활용한 영상 속 특정 색상 탐색 감지

TWELVE_LABS_API_KEY=<YOUR API KEY>
TWELVE_LABS_INDEX_ID=<YOUR_INDEX_ID>

yarn add twelvelabs-js # or npm i twelvelabs-js

const client = new TwelveLabs({ apiKey: API_KEY });

"use strict";


const express = require("express");
const dotenv = require("dotenv");
const cors = require("cors");
const asyncHandler = require("express-async-handler");
const fs = require("fs");
const path = require("path");
const { TwelveLabs } = require("twelvelabs-js");


dotenv.config();


const app = express();


app.use(express.json());
app.use(cors());
app.use(express.static(path.join(__dirname, "../frontend/public")));


const PORT = 5001;
const API_KEY = process.env.TWELVE_LABS_API_KEY;
const INDEX_ID = process.env.TWELVE_LABS_INDEX_ID;


const client = new TwelveLabs({ apiKey: API_KEY });


app.listen(PORT, () => console.log(`Server running on port ${PORT}`));

/** Get videos */
app.get(
 "/videos",
 asyncHandler(async (req, res, next) => {
   const { page_limit, page } = req.query;


   const videosResponse = await client.index.video.listPagination(INDEX_ID, {
     pageLimit: page_limit,
     page: page,
   });


   const videos = videosResponse.data.map((video) => ({
     id: video.id,
     metadata: video.metadata,
   }));


   res.json({
     videos,
     page_info: videosResponse.pageInfo,
   });
 })
);

videosResponse= VideoListWithPagination {  
  ...,
  data: [
    Video {
      _resource: [Video],
      _indexId: '...',
      id: '...',
      metadata: [Object],
      hls: undefined,
      source: undefined,
      indexedAt: '2024-06-27T05:11:29Z',
      createdAt: '2024-06-27T05:01:35Z',
      updatedAt: '2024-06-27T05:01:52Z'
    },
  	 ...
       ],
  pageInfo: {
    page: 1,
    limitPerPage: 12,
    totalPage: 3,
    totalResults: 29,
    totalDuration: 19122
  }
}

/** Get a video of an index */
app.get(
 "/videos/:videoId",
 asyncHandler(async (req, res, next) => {
   const { videoId } = req.params;


   const videoResponse = await client.index.video.retrieve(INDEX_ID, videoId);


   res.json({
     metadata: videoResponse.metadata,
     hls: videoResponse.hls,
     source: videoResponse.source,
   });
 })
);

videoResponse= Video {
  ...,
  id: '...',
  metadata: {
    duration: 54,
    engine_ids: [ 'marengo2.6', 'pegasus1.1' ],
    filename: 'tirtir korean cushion review',
    fps: 30,
    height: 1280,
    size: 9601300,
    video_title: 'tirtir korean cushion review',
    width: 720
  },
  hls: {
    videoUrl: '... .m3u8',
    thumbnailUrls: ['... .jpg'],
    status: 'COMPLETE',
    updatedAt: '2024-05-22T02:49:49.074Z'
  },
  source: {
    type: 'youtube',
    name: 'theoliviasaurusrex',
    url: 'https://www.youtube.com/watch?v=tOabvdtTa-U'
  },
  indexedAt: '2024-05-22T03:03:53Z',
  createdAt: '2024-05-22T02:49:28Z',
  updatedAt: '2024-05-22T02:49:36Z'
}

/** Search videos based on an image query */
app.get(
 "/search",
 asyncHandler(async (req, res, next) => {
   const { imageSrc, threshold, pageLimit, adjustConfidenceLevel } = req.query;


   const imagePath = path.join(
     __dirname,
     "../frontend/public/images",
     imageSrc
   );


   if (!fs.existsSync(imagePath)) {
     console.error("Image not found at path:", imagePath);
     return res.status(404).json({ error: "Image not found" });
   }


   const searchResponse = await client.search.query({
     indexId: INDEX_ID,
     queryMediaFile: fs.createReadStream(imagePath),
     queryMediaType: "image",
     options: ["visual"],
     threshold: threshold,
     pageLimit: pageLimit,
     adjustConfidenceLevel: adjustConfidenceLevel,
   });


   res.json({
     searchResults: searchResponse.data,
     pageInfo: searchResponse.pageInfo,
   });
 })
);

searchResponse= SearchResult {
  ...,
  pool: {
    totalCount: 29,
    totalDuration: 19122,
    indexId: '...'
  },
  data: [
    {
      score: 84.45,
      start: 379.13333333341933,
      end: 381,
      metadata: [Array],
      videoId: '...',
      confidence: 'high',
      thumbnailUrl: '...'
    },
      ...
      ],
  pageInfo: {
    limitPerPage: 12,
    totalResults: 20,
    pageExpiredAt: '2024-08-15T04:09:46Z'
    nextPageToken: '...' //This might not exist 
  }
}

/** Get search results of a specific page */
app.get(
 "/search/:pageToken",
 asyncHandler(async (req, res, next) => {
   const { pageToken } = req.params;


   let searchByPageResponse = await client.search.byPageToken(`${pageToken}`);


   res.json({
     searchResults: searchByPageResponse.data,
     pageInfo: searchByPageResponse.pageInfo,
   });
 })
);

async function showVideos(page = 1) {
 videoList.innerHTML = "";


 ...


 try {
   const { videosDetail, pageInfo } = await getVideoOfVideos(page);


   videoListLoading.removeChild(loadingSpinnerContainer);


   if (videosDetail) {
     videosDetail.forEach((video) => {
       const videoContainer = createVideoContainer(video);
       videoList.appendChild(videoContainer);
     });


     videoListLoading.classList.remove("min-h-[300px]");


     createPaginationButtons(pageInfo, page);
   }
 } catch (error) {
   console.error("Error fetching videos:", error);
 }
}

async function getVideoOfVideos(page = 1) {
 const videosResponse = await getVideos(page);


 if (videosResponse.videos.length > 0) {
   const videosDetail = await Promise.all(
     videosResponse.videos.map((video) => getVideo(video.id))
   );


   return { videosDetail, pageInfo: videosResponse.page_info };
 }
}

function createPageButton(pageNumber, currentPage) {
 const pageButton = document.createElement("button");
 pageButton.textContent = pageNumber;
  
  ...


 if (pageNumber === currentPage) {
   pageButton.classList.add("bg-slate-200", "font-medium");
   pageButton.disabled = true;
 } else {
   pageButton.classList.add("bg-transparent");
   pageButton.addEventListener("click", () => showVideos(pageNumber));
 }


 return pageButton;
}

async function handleSearchButtonClick() {
 toggleSearchButton(false);


 nextPageToken = null;
 searchResultContainer.innerHTML = "";
 videoListContainer.classList.add("hidden");
 searchResultContainer.classList.remove("hidden");
 searchResultList.innerHTML = "";


 const loadingSpinnerContainer = createLoadingSpinner();
 searchResultContainer.appendChild(loadingSpinnerContainer);


 try {
   const { searchResults } = await searchByImage();


   searchResultContainer.removeChild(loadingSpinnerContainer);


   if (searchResults.length > 0) {
     showSearchResults(searchResults);
   } else {
     displayNoResultsMessage();
   }
 } catch (error) {
   console.error("Error fetching search results:", error);
 } finally {
   toggleSearchButton(true);
 }
}

function createShowMoreButton() {
 removeExistingButton();


 const showMoreButtonContainer = document.createElement("div");
 showMoreButtonContainer.id = "show-more-button";
 showMoreButtonContainer.classList.add("flex", "justify-center", "my-4");


 const showMoreButton = document.createElement("button");
 showMoreButton.innerHTML = '<i class="fa-solid fa-chevron-up"></i> Show More';


 showMoreButton.addEventListener("click", async () => {
   const loadingSpinnerContainer = createLoadingSpinner();
   searchResultContainer.appendChild(loadingSpinnerContainer);


   const nextPageResults = await getNextSearchResults(nextPageToken);


   searchResultContainer.removeChild(loadingSpinnerContainer);


   showSearchResults(nextPageResults.searchResults);
   nextPageToken = nextPageResults.pageInfo.nextPageToken || null;
 });
 showMoreButtonContainer.appendChild(showMoreButton);
 searchResultContainer.appendChild(showMoreButtonContainer);
}