Run SkyReels for Image2Video Generation

Here, we show how to use Skywork/SkyReels-V1-Hunyuan-I2V Model for video generation from image.

Install and Run Inference

git clone https://github.com/SkyworkAI/SkyReels-V1
cd skyreelsinfer
pip install -r requirements.txt

#./SkyReels-V1/scripts/test_skyreel_image2video.py

import argparse
import sys
import time
import os
import random
sys.path.append("..")
from skyreelsinfer import TaskType
from skyreelsinfer.offload import OffloadConfig
from skyreelsinfer.skyreels_video_infer import SkyReelsVideoInfer
from diffusers.utils import export_to_video
from diffusers.utils import load_image

def get_transformer_model_id(task_type:str) -> str:
    return "Skywork/SkyReels-V1-Hunyuan-I2V" if task_type == "i2v" else "Skywork/SkyReels-V1-Hunyuan-T2V"

class SkyReelsImage2Video:
    def __init__(self, task_type='i2v', gpu_num=1):
        self.task_type = task_type
        self.predictor = SkyReelsVideoInfer(
                        task_type= TaskType.I2V if task_type == "i2v" else TaskType.T2V,
                        model_id=get_transformer_model_id(task_type),
                        quant_model=True,
                        world_size=gpu_num,
                        is_offload=True,
                        offload_config=OffloadConfig(
                            high_cpu_memory=True,
                            parameters_level=True,
                            compiler_transformer=False,
                            )
                        )

    def generate_video(self, prompt, image_path, save_dir='../../skyreel_videos', seed=-1):
        if seed == -1:
            random.seed(time.time())
            seed = int(random.randrange(4294967294))

        # print(f"image:{type(image_path)}")

        kwargs = {
            "prompt": prompt,
            "height": 512,
            "width": 512,
            "num_frames": 97,
            "num_inference_steps": 30,
            "seed": seed,
            "guidance_scale": 6.0,
            "embedded_guidance_scale": 1.0,
            "negative_prompt": "Aerial view, aerial view, overexposed, low quality, deformation, a poor composition, bad hands, bad teeth, bad eyes, bad limbs, distortion",
            "cfg_for": False,
        }

        if self.task_type == "i2v":
            assert image_path is not None, "please input image_path"
            kwargs["image"] = load_image(image=image_path)

        output = self.predictor.inference(kwargs)

        video_out_file = f"{save_dir}/{prompt[:5].replace('/','')}_{seed}.mp4"
        export_to_video(output, video_out_file, fps=24)
        print(f"generate video, local path: {video_out_file}")
        return video_out_file

if __name__ == '__main__':
    image2video_convert = SkyReelsImage2Video()
    prompt = 'The person slight move and trun around her/his body to show case the detial of cloth.'
    image_path = '../../user_data/model_images/model_2.jpg'
    video_path = image2video_convert.generate_video(prompt, image_path)