| Version 15 (modified by jazz, 16 years ago) (diff) |
|---|
Xen GPU Cluster Practice
實作三: 在虛擬機器上獲得 GPU 硬體加速 - 使用VMGL
步驟一: 在 Dom0 上安裝 VMGL
# 安裝環境所需套件.
rider@cloud:~/vmgl$ sudo apt-get install build-essential firefox flashplugin-nonfree htop mozilla-plugin-vlc libao2 libasound2 libmikmod2 mesa-common-dev libglu1-mesa-dev mesa-utils libxmu-headers libxmu6 libxmu-dev zlib1g-dev libjpeg62 libjpeg62-dev xutils-dev libxaw-headers libxaw7 libxaw7-dev libxext6 libxext-dev rxvt lwm vlc vlc-plugin-alsa tightvncserver xauth xvfb xfonts-100dpi xfonts-75dpi culmus xfonts-scalable xfonts-base
~$ sudo apt-get install build-essential firefox flashplugin-nonfree htop libao2 libasound2 libmikmod2 mesa-common-dev libglu1-mesa-dev mesa-utils libxmu-headers libxmu6 libxmu-dev zlib1g-dev libjpeg62 libjpeg62-dev xutils-dev libxext6 libxext-dev rxvt lwm tightvncserver xauth xvfb xfonts-100dpi xfonts-75dpi culmus xfonts-scalable xfonts-base
# 下載VMGL.
rider@cloud:~/vmgl$ wget http://www.cs.toronto.edu/~andreslc/software/vmgl-0.1.tar.bz2
rider@cloud:~/vmgl$ tar jxvf vmgl-0.1.tar.bz2
rider@cloud:~/vmgl$ cd ./vmgl.hg/tightvnc/
# Patch for AMD64_Machine.
rider@cloud:~/vmgl$ wget --no-check-certificate https://trac.nchc.org.tw/grid/attachment/wiki/Xen_GPU_cluster/tightvnc-1.2.9-amd64support.patch?format=raw
rider@cloud:~/vmgl$ mv tightvnc-1.2.9-amd64support.patch?format=raw tightvnc-1.2.9-amd64support.patch
rider@cloud:~/vmgl/vmgl.hg/tightvnc$ patch -p0 < ../../tightvnc-1.2.9-amd64support.patch
rider@cloud:~/vmgl$ cd ..
# 設定編譯器路徑.
rider@cloud:~/vmgl$ sudo ln -sf /usr/bin/make /usr/bin/gmake (if necessary)
# 編譯.
rider@cloud:~/vmgl/vmgl.hg$ make -j 4
~/vmgl/vmgl.hg$ sudo make
# 安裝於 Dom0(host) 端上.
rider@cloud:~/vmgl/vmgl.hg$ sudo make install-host
# 設定主機信任清單.
rider@cloud:~$ xauth
# Set the authority for remote guest. Using authority file /home/rider/.Xauthority xauth> add guest/unix:10 MIT-MAGIC-COOKIE-1 ec0ffd387888b9749d55f88031505888 xauth> add guest/unix:1 MIT-MAGIC-COOKIE-1 6824789b4ce0ac5743aeb57fd3ef8f5b xauth> exit
rider@cloud:~$ xauth list
guest:1 MIT-MAGIC-COOKIE-1 5c53c8c640b816d7714e23d5cfcd9a8e cloud/unix:1 MIT-MAGIC-COOKIE-1 5c53c8c640b816d7714e23d5cfcd9a8e guest:2 MIT-MAGIC-COOKIE-1 5bfe43007be14ca93e9ee5fc71701463 cloud/unix:2 MIT-MAGIC-COOKIE-1 5bfe43007be14ca93e9ee5fc71701463 guest:3 MIT-MAGIC-COOKIE-1 04499062b48a199921e859ee76d267ab cloud/unix:3 MIT-MAGIC-COOKIE-1 04499062b48a199921e859ee76d267ab guest/unix:10 MIT-MAGIC-COOKIE-1 ec0ffd387888b9749d55f88031505888 guest/unix:1 MIT-MAGIC-COOKIE-1 6824789b4ce0ac5743aeb57fd3ef8f5b guest/unix:2 MIT-MAGIC-COOKIE-1 98e85e7e551246d428023b54b5dff02b cloud/unix:0 MIT-MAGIC-COOKIE-1 b5f3b4b4f9d0e63e8f9b88a6d57fad15 localhost:0 MIT-MAGIC-COOKIE-1 b5f3b4b4f9d0e63e8f9b88a6d57fad15
步驟二: 建構一台 VMGL Guest VM
# 設定你想要怎樣規格的虛擬機器.
rider@cloud:~$ sudo vim /etc/xen-tools/xen-tools.conf
dir = /home install-method = debootstrap size = 8Gb # Disk image size. memory = 1024Mb # Memory size swap = 128Mb # Swap size fs = ext3 # use the EXT3 filesystem for the disk image. dist = hardy # Default distribution to install. ---> For CUDA Support (Ubuntu 8.0.4) image = sparse # Specify sparse vs. full disk images. gateway = 140.XXX.XXX.XXX netmask = 255.255.255.0 broadcast = 140.XXX.XXX.XXX kernel = /boot/vmlinuz-`uname -r` initrd = /boot/initrd.img-`uname -r` mirror = http://gb.archive.ubuntu.com/ubuntu/ ext3_options = noatime,nodiratime,errors=remount-ro ext2_options = noatime,nodiratime,errors=remount-ro xfs_options = defaults reiser_options = defaults
rider@cloud:~$ sudo xen-create-image --hostname guest --ip 140.XXX.XXX.XXX
步驟三: 設定虛擬機器(Guest)
rider@cloud:~$ sudo vim /etc/xen/nvidia.cfg
kernel = '/boot/vmlinuz-2.6.22.9'
ramdisk = '/boot/initrd.img-2.6.22.9'
memory = '1024'
vcpus = '4'
# 不需配置 PCIE 顯示卡
# pci = ['01:00.0']
root = '/dev/sda2 ro'
disk = [
'file:/home/domains/nvidia/disk.img,sda2,w',
'file:/home/domains/nvidia/swap.img,sda1,w',
]
name = 'nvidia'
#
# Networking
#
vif = [ 'ip=140.xxx.xxx.xxx,mac=00:16:3E:AA:70:5C' ]
#
# Behaviour
#
on_poweroff = 'destroy'
on_reboot = 'restart'
on_crash = 'restart'
步驟四: 在 DomU(Guest) 上安裝 VMGL
# 啟動虛擬機器Guest並登入.
# 方法一.
rider@cloud:~$ sudo xm create -c guest.cfg
# 方法二.
@Dom0
rider@cloud:~$ sudo xm create guest.cfg
@ClientUser
rider@PC:~$ ssh 140.xxx.xxx.xxx
# 安裝環境所需套件.
rider@guest:~$ mkdir -p vmgl
rider@guest:~$ cd ./vmgl
rider@guest:~/vmgl$ sudo apt-get install build-essential firefox flashplugin-nonfree htop mozilla-plugin-vlc libao2 libasound2 libmikmod2 mesa-common-dev libglu1-mesa-dev mesa-utils libxmu-headers libxmu6 libxmu-dev zlib1g-dev libjpeg62 libjpeg62-dev xutils-dev libxaw-headers libxaw7 libxaw7-dev libxext6 libxext-dev rxvt lwm vlc vlc-plugin-alsa tightvncserver xauth xvfb xfonts-100dpi xfonts-75dpi culmus xfonts-scalable xfonts-base
# 基本桌面環境.
# Desgin for lightweight basic windows support xfonts-base xfonts-100dpi xfonts-75dpi -> fonts support rxvt -> x-terminal-emulator for VNC lwm -> x-window-manager for VNC
# 下載VMGL.
rider@guest:~/vmgl$ wget http://www.cs.toronto.edu/~andreslc/software/vmgl-0.1.tar.bz2
rider@guest:~/v,gl$ wget http://trac.nchc.org.tw/grid/raw-attachment/wiki/Xen_GPU_cluster/tightvnc-1.2.9-amd64support.patch
rider@guest:~/vmgl$ tar jxvf vmgl-0.1.tar.bz2
rider@guest:~/vmgl$ cd ./vmgl.hg/tightvnc/
# Patch for AMD64_Machine.
rider@guest:~/vmgl/vmgl.hg/tightvnc$ patch -p0 < ../../tightvnc-1.2.9-amd64support.patch
rider@guest:~/vmgl$ cd ..
# 設定編譯器路徑.
rider@guest:~/vmgl$ sudo ln -sf /usr/bin/make /usr/bin/gmake (if necessary)
# 建立安裝所需基本目錄.
rider@guest:~/vmgl/vmgl.hg$ sudo mkdir -p /usr/lib/xorg (if necessary)
rider@guest:~/vmgl/vmgl.hg$ sudo mkdir -p /usr/lib/xorg/modules (if necessary)
rider@guest:~/vmgl/vmgl.hg$ sudo mkdir -p /usr/lib/xorg/modules/extensions (if necessary)
# 編譯.
rider@guest:~/vmgl/vmgl.hg$ make
# 安裝於 DomU(guest) 端上.
rider@guest:~/vmgl/vmgl.hg$ sudo make install-guest
# 修正 rgb PATH 問題.
## Fix the rgb_Path problem. rider@guest:~/vmgl/vmgl.hg$ sudo mkdir -p /usr/X11R6/lib rider@guest:~/vmgl/vmgl.hg$ sudo mkdir /usr/X11R6/lib/X11 rider@guest:~/vmgl/vmgl.hg$ sudo ln -sf /etc/X11/rgb.txt /usr/X11R6/lib/X11/rgb
步驟五: VMGL 使用說明: 在Dom0 上的設定
# 將 X-Windows 桌面 export 出來.
rider@cloud:~$ export DISPLAY=:0
# 啟動 VLGL stub-daemon.
rider@cloud:~$ stub-daemon
# 檢查 stub-daemon 所對應的 port(7000) 有無開啟.
rider@cloud:~$ netstat -tunlp
Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:8002 0.0.0.0:* LISTEN - tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN - tcp 0 0 127.0.0.1:631 0.0.0.0:* LISTEN - tcp 0 0 0.0.0.0:7000 0.0.0.0:* LISTEN 29082/stub-daemon ---> VMGL stub-daemon udp 0 0 0.0.0.0:32769 0.0.0.0:* - udp 0 0 0.0.0.0:5353 0.0.0.0:* -
步驟六: VMGL 使用說明: 在DomU 上的設定
# 設定 DomU 上的字型路徑對應.
rider@guest:~$ sudo ln -sf /usr/share/fonts/X11/ /usr/X11R6/lib/X11/fonts (if necessary - fix font path)
# FontPath: /usr/X11R6/lib/X11/fonts or /usr/share/fonts/X11
# 確認 rgb 存在於對應路徑.
rider@guest:~$ less /usr/X11R6/lib/X11/rgb.txt (if necessary - rgb path confirmation)
# 設定 DomU 上的 VMGL 環境變數.
rider@guest:~$ sudo vim /etc/profile
GLSTUB=Cloud_IP(host_IP):7000 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/vmgl LD_PRELOAD=/usr/local/lib/vmgl/libGL.so export GLSTUB LD_LIBRARY_PATH LD_PRELOAD
rider@guest:~$ source /etc/profile
# 設定 DomU 上的 X forwarding.
Using X forwarding
rider@guest:~$ sudo vim /etc/ssh/sshd_config
X11Forwarding yes
rider@guest:~$ sudo vim /etc/ssh/ssh_config
ForwardX11 yes
ForwardX11Trusted yes
# 重新啟動 DomU 上的 SSH Service.
rider@guest:~$ sudo /etc/init.d/ssh restart
# 設定 VNC server 啟動參數.
rider@guest:~$ vim .vnc/xstartup
# xrdb $HOME/.Xresources
# 設定 VNC server 使用者密碼.
rider@guest:~$ vncpasswd
# 開啟 DomU 上的第一台VNC Server.
rider@guest:~$ vncserver -geometry 1024x768 -depth 24 :1
# 開啟 DomU 上的第二台VNC Server.(可跳過此步驟)
rider@guest:~$ vncserver -geometry 1024x768 -depth 24 :2
# 檢查 VNCserver 有無開啟對應之 port(5901開始).
rider@guest:~$ netstat -tunlp
(Not all processes could be identified, non-owned process info will not be shown, you would have to be root to see it all.) Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 0.0.0.0:5901 0.0.0.0:* LISTEN 10652/Xtightvnc ---> guest:1 tcp 0 0 0.0.0.0:5902 0.0.0.0:* LISTEN 10630/Xtightvnc tcp 0 0 0.0.0.0:6001 0.0.0.0:* LISTEN 10652/Xtightvnc ---> guest:2 tcp 0 0 0.0.0.0:6002 0.0.0.0:* LISTEN 10630/Xtightvnc tcp 0 0 127.0.0.1:6010 0.0.0.0:* LISTEN - tcp6 0 0 :::22 :::* LISTEN - tcp6 0 0 ::1:6010 :::* LISTEN -
# 將 VNC Server:1 export 出來.
rider@guest:~$ export DISPLAY=:1
步驟七: 在 Dom0 上 透過 X forwarding 登入 DomU
# 使用 X forwarding 的方式登入 DomU.
Using X forwarding
rider@cloud:~$ ssh -X guest
# 檢查 direct rendering 有無起來.
rider@guest:~$ glxinfo
name of display: localhost:10.0
display: localhost:10 screen: 0
direct rendering: Yes
server glx vendor string: VMGL
server glx version string: 1.2 VMGL
server glx extensions:
client glx vendor string: VMGL
client glx version string: 1.2 VMGL
client glx extensions:
GLX version: 1.3
GLX extensions:
OpenGL extensions:
GL_ARB_depth_texture, GL_ARB_fragment_program, GL_ARB_multisample,
GL_ARB_multitexture, GL_ARB_occlusion_query, GL_ARB_point_parameters,
GL_ARB_point_sprite, GL_ARB_shadow, GL_ARB_texture_border_clamp,
GL_ARB_texture_compression, GL_ARB_texture_cube_map,
GL_ARB_texture_env_add, GL_ARB_texture_env_combine,
GL_EXT_texture_env_combine, GL_ARB_texture_env_dot3,
GL_EXT_texture_env_dot3, GL_ARB_texture_mirrored_repeat,
GL_ARB_texture_non_power_of_two, GL_ARB_transpose_matrix,
GL_ARB_vertex_buffer_object, GL_ARB_vertex_program, GL_ARB_window_pos,
GL_EXT_blend_color, GL_EXT_blend_minmax, GL_EXT_blend_func_separate,
GL_EXT_blend_subtract, GL_EXT_texture_env_add, GL_EXT_fog_coord,
GL_EXT_multi_draw_arrays, GL_EXT_secondary_color, GL_EXT_shadow_funcs,
GL_EXT_stencil_wrap, GL_EXT_texture_cube_map, GL_EXT_texture_edge_clamp,
GL_EXT_texture_filter_anisotropic, GL_EXT_texture_lod_bias,
GL_EXT_texture_object, GL_EXT_texture3D, GL_EXT_bgra,
GL_IBM_rasterpos_clip, GL_NV_fog_distance, GL_NV_fragment_program,
GL_NV_register_combiners, GL_NV_register_combiners2,
GL_NV_texgen_reflection, GL_NV_texture_rectangle, GL_NV_vertex_program,
GL_NV_vertex_program1_1, GL_NV_vertex_program2, GL_SGIS_generate_mipmap,
GL_CR_state_parameter, GL_CR_cursor_position, GL_CR_bounding_box,
GL_CR_print_string, GL_CR_tilesort_info, GL_CR_synchronization,
GL_CR_head_spu_name, GL_CR_performance_info, GL_CR_window_size,
GL_CR_tile_info, GL_CR_saveframe, GL_CR_readback_barrier_size,
GL_CR_server_id_sharing, GL_CR_server_matrix
GLX_ARB_multisample
OpenGL vendor string: H. Andres Lagar-Cavilla
OpenGL renderer string: VMGL
OpenGL version string: 1.5 VMGL 1.9
0x52 24 dc 0 32 0 r y . 8 8 8 0 4 24 8 16 16 16 16 2 1 None
0x53 24 dc 0 32 0 r y . 8 8 8 8 4 24 8 16 16 16 16 2 1 None
0x54 24 dc 0 32 0 r y . 8 8 8 0 4 24 8 16 16 16 16 4 1 None
0x55 24 dc 0 32 0 r y . 8 8 8 8 4 24 8 16 16 16 16 4 1 None
0x56 24 dc 0 32 0 r . . 8 8 8 0 4 24 8 16 16 16 16 2 1 None
0x57 24 dc 0 32 0 r . . 8 8 8 8 4 24 8 16 16 16 16 2 1 None
0x58 24 dc 0 32 0 r . . 8 8 8 0 4 24 8 16 16 16 16 4 1 None
0x59 24 dc 0 32 0 r . . 8 8 8 8 4 24 8 16 16 16 16 4 1 None
0x23 32 tc 0 32 0 r y . 8 8 8 0 4 24 8 16 16 16 16 0 0 None
0x5a 32 tc 0 32 0 r y . 8 8 8 8 4 24 8 16 16 16 16 0 0 None
0x5b 32 tc 0 32 0 r . . 8 8 8 0 4 24 8 16 16 16 16 0 0 None
0x5c 32 tc 0 32 0 r . . 8 8 8 8 4 24 8 16 16 16 16 0 0 None
0x5d 32 tc 0 32 0 r y . 8 8 8 0 4 24 0 16 16 16 16 0 0 None
0x5e 32 tc 0 32 0 r y . 8 8 8 8 4 24 0 16 16 16 16 0 0 None
0x5f 32 tc 0 32 0 r . . 8 8 8 0 4 24 0 16 16 16 16 0 0 None
0x60 32 tc 0 32 0 r . . 8 8 8 8 4 24 0 16 16 16 16 0 0 None
0x61 32 tc 0 32 0 r y . 8 8 8 0 4 0 0 16 16 16 16 0 0 None
0x62 32 tc 0 32 0 r y . 8 8 8 8 4 0 0 16 16 16 16 0 0 None
0x63 32 tc 0 32 0 r . . 8 8 8 0 4 0 0 16 16 16 16 0 0 None
0x64 32 tc 0 32 0 r . . 8 8 8 8 4 0 0 16 16 16 16 0 0 None
0x65 32 tc 0 32 0 r y . 8 8 8 0 4 24 0 16 16 16 16 2 1 None
0x66 32 tc 0 32 0 r y . 8 8 8 8 4 24 0 16 16 16 16 2 1 None
0x67 32 tc 0 32 0 r y . 8 8 8 0 4 24 0 16 16 16 16 4 1 None
0x68 32 tc 0 32 0 r y . 8 8 8 8 4 24 0 16 16 16 16 4 1 None
0x69 32 tc 0 32 0 r . . 8 8 8 0 4 24 0 16 16 16 16 2 1 None
0x6a 32 tc 0 32 0 r . . 8 8 8 8 4 24 0 16 16 16 16 2 1 None
0x6b 32 tc 0 32 0 r . . 8 8 8 0 4 24 0 16 16 16 16 4 1 None
0x6c 32 tc 0 32 0 r . . 8 8 8 8 4 24 0 16 16 16 16 4 1 None
0x6d 32 tc 0 32 0 r y . 8 8 8 0 4 24 8 16 16 16 16 2 1 None
0x6e 32 tc 0 32 0 r y . 8 8 8 8 4 24 8 16 16 16 16 2 1 None
0x6f 32 tc 0 32 0 r y . 8 8 8 0 4 24 8 16 16 16 16 4 1 None
0x70 32 tc 0 32 0 r y . 8 8 8 8 4 24 8 16 16 16 16 4 1 None
0x71 32 tc 0 32 0 r . . 8 8 8 0 4 24 8 16 16 16 16 2 1 None
0x72 32 tc 0 32 0 r . . 8 8 8 8 4 24 8 16 16 16 16 2 1 None
0x73 32 tc 0 32 0 r . . 8 8 8 0 4 24 8 16 16 16 16 4 1 None
0x74 32 tc 0 32 0 r . . 8 8 8 8 4 24 8 16 16 16 16 4 1 None
# DomU 上的 glxgears 效能測試.
rider@guest:~$ glxgears
47819 frames in 5.0 seconds = 9563.678 FPS 46064 frames in 5.0 seconds = 9212.566 FPS 44584 frames in 5.0 seconds = 8916.581 FPS 44256 frames in 5.0 seconds = 8850.974 FPS 44688 frames in 5.0 seconds = 8937.528 FPS
步驟八: 使用 VNC Viewer 登入 DomU
# 透過 VNC Viewer(TurboVNC Viewer or Tightvnc viewer)登入 DomU 的 VNC Server,登入後便可以透過 rxvt 來下指令跑相關3D應用了.
@ ClientUser
rider@PC:~$ vncviewer guest:1
# 檢查 direct rendering=yes?.
@ VNC viewer 登入 :1 後
rider@guest:~$ glxinfo
實作四: 在虛擬機器上 - 體驗 3D 硬體加速
步驟九: 喀 Game 時間
# 開啟 Firefox 來玩 Doom Online.
rider@guest:~$ firefox http://necromanthus.com/Games/Flash/doom.html
步驟十: 賞 "片" 時間
rider@guest:~$ mkdir unigine
rider@guest:~$ mkdir ./unigine/avi
rider@guest:~$ cd ./unigine/avi
rider@guest:~/unigine/avi$ wget http://mail.pdaxrom.org/unigine.com/Unigine_Tropics_640x360.avi
rider@guest:~/unigine/avi$ wget http://unigine.com/download/video/Unigine_Tropics_1024x576.avi
# 在 rxvt 叫出 VLC 來播放測試影片.
rider@guest:~/unigine/avi$ vlc Unigine_Tropics_640x360.avi
# 有興趣的也可以在行動裝置上使用 VNC Viewer or VLC 來實驗看看.
步驟十一: Firefox Cooliris 嘗鮮
# 打開 Firefox 打開"工具" -> "附加元件" -> "取得元件" -> 搜尋並安裝"cooliris" -> 重新啟動 Firefox.
# 若有問題請執行以下步驟.
rider@guest:~$ mkdir -p .mozilla/plugins
# 下載 Adobe 64-bit Flash Player 10 plugin.
rider@guest:~$ wget http://download.macromedia.com/pub/labs/flashplayer10/libflashplayer-10.0.22.87.linux-x86_64.so.tar.gz
rider@guest:~$ tar -zxvf libflashplayer-10.0.22.87.linux-x86_64.so.tar.gz -C ~/.mozilla/plugins
# Check 有無遺失 Librady 或路徑有誤.
rider@guest:~$ ldd ~/.mozilla/firefox/xc8lwx2j.default/extensions/piclens@cooliris.com/libs-64/libcooliris19.so
rider@guest:~$ ldd ~/.mozilla/firefox/xc8lwx2j.default/extensions/piclens@cooliris.com/libs-64/libav.so
# 安裝所需 Library , 並建立 linking.
rider@guest:~$ sudo apt-get install libmozjs0d libmozillainterfaces-java xulrunner-1.9 libxsltc-java libxslt1.1 xulrunner
rider@guest:~$ sudo ln -sf /usr/lib/libxpcom.so.0d /usr/lib/libxpcom.so
rider@guest:~$ sudo ln -sf /usr/lib/libmozjs.so.0d /usr/lib/libmozjs.so
rider@guest:~$ sudo ldconfig
# Restart Firefox.
步驟十二: Dom0 OpenGL 3.0 效能實測
# 使用工具: "Unigine Tropics" or "NVIDIA PerfKit" 來測試 NVIDIA OpenGL 3.0 效能
# 使用 Tropics.
rider@cloud:~$ mkdir unigine
rider@cloud:~$ cd ./unigine
rider@cloud:~/unigine$ wget http://unigine.com/download/files/Unigine_Tropics-1.1.tar.bz2
rider@cloud:~/unigine$ mv Unigine_Tropics-1.1.tar.bz2\?noredirect Unigine_Tropics-1.1.tar.bz2
rider@cloud:~/unigine$ tar jxvf Unigine_Tropics-1.1.tar.bz2
rider@cloud:~/unigine$ cd ./Unigine_Tropics-1.1/
# 播放 1024x768 的影片來作 Benchmark.
rider@cloud:~/unigine/Unigine_Tropics-1.1$ ./windowed_1024x768.sh
