ft mpi europvm03 ijsa

                        !"# $% & '# ( )*+ )& ,-( .((-/0 /! 1-%230 ( 4-0-5# 1-%230! 6* !(*! ,!20!%!(0# 7( /!8 05 -9 :!((!88!!# ;;<< =-"3(0!! >"/&# 63 0! ?;@# A(-B/ ""!# :CD@EFFG# 76H I! !"# 9# -(JK*8&30+&!3 C-/!%! <<#
dc _UZ ghiel WXZ`Vm`Y_Vdap kZ kd^]b ]V‰Z _d Va\ZW_Ve jY_Z Va _UVW XYXZ[p kUY_ _UZ XZ[cd[ f Ya`Z sZaZm_W Yab b[YksY`‰W dc bVuZ[Za_ cZY_^[ZW dc ghiel Y[Zn Tkd ² ^ZW_VdaW Y[Z dc WXZ`Vm` Va_Z[ZW_ Va _UZ `da_Z_ dc _UVW XYXZ[³ m[W_p bd bqaY f V`Y]]q `[ZY_Zb `d ff ^aV`Y_d[W duZ[ _UZ WY f Z XdVa_e_deXdVa_ XZ[cd[ f Ya`Z da `^[[Za_ Vf X]Z f Za_Y_VdaW YW _UZ W_Y_V` ghi ˆ‡gg v ‡´µ¶ YXX[dY`U· „ab WZ`dabp kUY_ VW _UZ Y`UVZ\Ys]Z XZ[e cd[ f Ya`Z ^WVaj daZeWVbZb dXZ[Y_VdaW· ¸Va`Z _UZ a^ f sZ[ dc Y\YV]Ys]Z ghi Vf X]Z f Za_Ye _VdaW Vf X]Z f Za_Vaj Wd f Z XY[_W dc _UZ ghiel WXZ`Ve m`Y_Vda VW f ZYakUV]Z ² ^V_Z ]Y[jZp _UVW XYXZ[ `Ya ad_ jV\Z Y `d f X]Z_Z d\Z[\VZk Ysd^_ Y]] `^[[Za_]q Y\YV]e Ys]Z ghie]Vs[Y[VZW Yab _UZ cZY_^[ZW _UZq Y[Z X[d\Vbe Vajn o^[_UZ[ f d[Zp V_ VW ad_ _UZ Va_Za_Vda dc _UVW XYe XZ[ _d `d f XY[Z XZ[cd[ f Ya`Z [ZW^]_W sZ_kZZa _UZ f Ye `UVaZWp s^_ _d `d f XY[Z _UZ a^ f sZ[W Y`UVZ\Zb ^WVaj ghiel c^a`_VdaY]V_q kV_U _UZ XZ[cd[ f Ya`Z f ZYW^[Zb da _UZ \Z[q WY f Z f Y`UVaZ cd[ W_Y_V` ghiet W`ZaY[e VdWp Yab _UZ[Zcd[Z _d `d ff Za_ da _UZ ² ^Y]V_q dc _UZ Vf X]Z f Za_Y_Vda dc _UZ ghiel c^a`_VdaY]V_qn „W kV_U Y]] sZa`U f Y[‰ a^ f sZ[Wp _UZ [ZYbZ[ WUd^]b sZ YkY[Zp _UY_ Y]] [ZW^]_W [ZX[ZWZa_ da]q Y WaYXWUd_n TUZ W_[^`_^[Z dc _UVW XYXZ[ VW YW cd]]dkW³ Va WZ`e _Vda l kZ X[ZWZa_ s[VZ¹q _UZ _ZW_eW^V_Z ^WZb _U[d^jUe d^_ _UZ XYXZ[n ¸Z`_Vda  X[ZWZa_W _UZ [ZW^]_W Yab ZXZ[VZa`ZW kV_U UYab]Vaj bqaY f V` `d ff ^aV`Y_d[Wn ¸Z`_Vda ‹ X[ZWZa_W _UZ XZ[cd[ f Ya`Z Y`UVZ\Zb ^WVaj daZeWVbZb dXZ[Y_VdaW kV_U bVuZ[Za_ ghie]Vs[Y[VZW Yab

w xy PQz {|SP}z y TUZ ghiel WXZ`Vm`Y_Vda ~€ Z_ZabW _UZ ghiet bd`^e f Za_ ~l€ sq _U[ZZ f Y‚d[ `UYX_Z[Wp WZ\Z[Y] f Vad[ daZWp Yab Wd f Z `d[[Z`_VdaWƒ`]Y[Vm`Y_VdaW cd[ ghiet c^a`e _VdaWn „]_Ud^jU V_ UYW sZZa X^s]VWUZb WVa`Z t……†p `^[[Za_]q da]q _UZ XY[Y]]Z] m]Zeiƒ‡ `UYX_Z[ UYW sZZa Y``ZX_Zb sq _UZ Zabe^WZ[Wn ˆ]ZY[]q _UVW VW _UZ [ZYe Wdap _UY_ bZWXV_Z dc UY\Vaj f Yaq sZa`U f Y[‰W _ZW_Vaj ghiet c^a`_VdaY]V_q ~…p tŠp tt€p ghiel c^a`_VdaY]V_q UYW da]q sZZa Z\Y]^Y_Zb ^X _d adk Va _UVW Y[ZY ~‹€n „WW^ f Vaj _UY_ _UZ ^WZ[ [ZY]]q kYa_W _d ^WZ cZY_^[ZW ŒŽ ‘’“” ‘• –——’“˜™š ›œ ˜Ž™ ž Ÿ™—•“˜ ™¡˜ ’¢ £¡™“¤œ ˜Ž“’–¤Ž ¥’¡˜“•¥˜ ¡– ›™“ Ÿ£¦§¨©ª¦««£¬ª­®¯°± t

bVW`^WWZW Wd f Z [Z]Y_Zb VWW^ZWn oVaY]]qp WZ`_Vda W^ f e YW WVf X]Z X[dj[Y f W kUV`U ZaYs]Z `d f XY[Vaj WZ\Z[Y] f Y[VrZW _UZ [ZW^]_W Y`UVZ\Zb Yab X[ZWZa_W _UZ dajdVaj f ZYW^[Z f Za_Wn kd[‰ Va _UVW Y[ZYn od[ ZY`U s^V]bVaj s]d`‰p WZ\Z[Y] [ZcZ[Za`Z f db^]ZW Y[Z Y\YV]Ys]Zn „W Ya ZY f X]Zp WZ\Z[Y] `daW_[^`_d[W cd[ bZ[V\Zb bY_Y_qXZW Y[Z X[d\VbZb YW kZ]] YW bVuZ[Za_
    RP  y S  P  OP  O|}P  bY_Y _[YaWcZ[ X[Vf V_V\ZW d[ `d ff ^aV`Y_d[ `daW_[^`e _d[Wn od[ _UZ YaY]qWVW X[ZWZa_Zb Va _UVW XYXZ[p aZk TUVW WZ`_Vda jV\ZW Y s[VZc Va_[db^`_Vda _d _UZ ]Y_Za`q `d ff ^aV`Y_d[ `daW_[^`_d[W ^WVaj _UZ f Z_UdbW X[de _ZW_eW^V_Zp kUV`U UYW sZZa ^WZb cd[ _UZ XZ[cd[ f Ya`Z \VbZb Va _UZ ghiel UY\Z sZZa YbbZbp Z\Y]^Y_Vda _U[d^jUd^_ _UVW XYXZ[n TUZ ]Y_Za`q _ZW_e YW WUdka Va WZ`_Vda WXZ`Vm`Y_Vda YW kZ]] YW aZk bY_Y _[YaWcZ[ W^V_Z VW Y UVW_d[V`Y]]q j[dka `d]]Z`_Vda dc _ZW_Wp kUV`U X[Vf V_V\ZW ^WVaj daZeWVbZb `d ff ^aV`Y_Vdap YW WUdka UYW sZZa ^WZb _d f ZYW^[Z Y kVbZ \Y[VZ_q dc bVuZ[Za_ Va WZ`_Vda ‹n XZ[cd[ f Ya`Z `UY[Y`_Z[VW_V`W ~tlp t€n ´Z`Za_]qp _UZ _ZW_eW^V_Z UYW sZZa [ZeY[[YajZb W^`U _UY_ V_ X[d\VbZW s^V]bVaj s]d`‰W cd[ XdVa_e_deXdVa_ sZa`U f Y[‰Wp `[ZY_e   Q  zQ  R y S  Q  O| PO  }P  {   Vaj Ya ZYWV]q Z_ZabYs]Z sZa`U f Y[‰Vaj Za\V[da f Za_n y R  }S Sz   | y }SRPzQO  abe^WZ[W `Ya _U^W `[ZY_Z _UZV[ dka XdVa_e_deXdVa_ sZa`U f Y[‰ kUV`U Va`d[Xd[Y_ZW [Z]Z\Ya_ cZY_^[ZW dc ghiet WXZ`Vm`Y_Vda ZX]V`V_]q [ZW_[V`_Zb V_WZ]c da _UZV[ YXX]V`Y_Vda Yab _U^W Z\Y]^Y_Z _UZ XZ[cd[ f Ya`Z TUZ Y W_Y_V` j[d^X dc X[d`ZWWZWp Y[j^Vaj Y f dajW_ d_Ue dc _UZV[ bY_Y Z`UYajZ [d^_VaZWn Z[W  p _UY_ da f dW_ X]Y_cd[ f W _UZ XZ[cd[ f Ya`Z dc ghi „ f daj _UZ s^V]bVaj s]d`‰W Y[Z³ kV]] sZ WVjaVm`Ya_]q sZ__Z[ bZY]Vaj kV_U Y `daW_Ya_ X[d`ZWW j[d^Xn TUZ f d_V\Y_Vda sZUVab _UZ Va\ZW_VjYe \Y[VYs]Z `d ff ^aV`Y_d[ Y[j^ f Za_Wp _VdaW Va _UVW `UYX_Z[ _UZ[Zcd[Z VW cd`^WZb Y[d^ab _UZ \Y[VYs]Z bY_Y _qXZWp Va`]^bVaj ^WZ[ bZmaZb bY_Y ² ^ZW_Vdap kUZ_UZ[ _UVW `d ff da YWW^ f X_Vda VW ‚^We _qXZWp _VmZbn gd[Z WXZ`Vm`Y]]qp Y[Z _UZ UVjU XZ[cd[ f Ya`Z aZ_kd[‰Vaj bZ\V`ZW ^WZb da f dbZ[a Y[`UV_Z`_^[ZW `Ye \Y[VYs]Z `d ff ^aV`Y_Vda XY[_aZ[Wp XYs]Z dc bqaY f V`Y]]q ZW_Ys]VWUVaj `daaZ`_VdaW _d aZk X[d`ZWWZWp d[ Y[Z ghi ]Vs[Y[VZW Vf X]Z f Za_Vaj _UVW XY[_ \Y[VYs]Z bY_Y _[YaWcZ[ X[V V_V\ZWp `^[[Za_]q [Ze f dc _UZ ghiel WXZ`Vm`Y_Vda cY]]Vaj sY`‰ _d W]dkZ[ bZe W_[V`_Zb UdkZ\Z[ _d XVajeXdaj sZa`U f Y[‰Wn Znjn Tˆhƒih  · TUZ _ZW_eW^V_Z ^WZW _kd bVuZ[Za_ f Z_UdbW cd[ bZe \V`ZW TUZ bd`^ f Za_ jV\ZW _UZ ^WZ[ _U[ZZ XdWWVe _Z[ f VaVaj _UZ aZ_ f ZWWYjZ WVrZ _d sZ f ZYW^[Zb³ Y sV]V_VZW ghiel da Udk _d `[ZY_Z Y aZk `d ^aV`Y_d[ _UY_ f ^]_VX]V`Y_V\Z Va`[ZYWZ cd[ WUd[_ f ZWWYjZ ]Zaj_Up Yab Va`]^bZW X[d`ZWWZWp kUV`U UY\Z ad_ ff sZZa XY[_ dc _UZ Ya YbbV_V\Z Va`[ZYWZ dc _UZ f ZWWYjZ WVrZ cd[ ]Y[jZ f ZWe X[Z\Vd^W kd[]bej[d^X³ WYjZWn TUVW ZaYs]ZW Y bZ_YV]Zb YaY]qWVW dc WUd[_ f ZWe WYjZ sZUY\Vd[ YW kZ]] YW Y [ZYWdaYs]Z a^ f sZ[ dc f ZYe tn ¸XYka YbbV_VdaY] X[d`ZWWZW ^WVaj ghi ˆd ff e W^[Z f Za_W cd[ ]Y[jZ f ZWWYjZWn WXYkan TUZ `UV]b X[d`ZWWZW `Ya [Z_[VZ\Z _UZ TUZ ]Y_Za`q _ZW_eW^V_Z [ZXd[_W cd[ ZY`U f ZWWYjZ `d ff da `d ff ^aV`Y_d[ sq `Y]]Vaj ghi ˆd ff e WVrZ Ya Y\Z[YjZp f YVf ^ f Yab f VaVf ^ f sYabkVb_U jZ_ XY[Za_n Y`UVZ\Zb Yab _UZ Y``d[bVaj ZZ`^_Vda _Vf Zn o^[_UZ[e f d[Zp _UZ W_YabY[b bZ\VY_Vda VW [ZXd[_Zb cd[ ZY`U ln ˆdaaZ`_ _kd Y][ZYbq [^aaVaj XY[Y]]Z]  YXX]V`Ye _VdaW ^WVaj Y Wd`‰Z_e]V‰Z Va_Z[cY`Zn ‡aZ YXX]V`Ye f ZWWYjZ WVrZp VabV`Y_Vaj _UZ W_YsV]V_q dc _UZ f ZYW^[Ze Za_n TUZ d^_X^_ `Ya ZV_UZ[ sZ k[V__Za _d W_YabY[b _Vda duZ[W Y WZ[\V`Z  ^WVaj ghi ˆd ff Y``ZX_p f d^_X^_ d[ _d Y m]Zp ^WVaj ZV_UZ[ W_YabY[b  i m]Z kUV]Z Yad_UZ[ YXX]V`Y_Vda ZW_Ys]VWUZW `daaZ`e dXZ[Y_VdaW d[ [Z]qVaj da ghieiƒ‡n ¸Y f X]Z ja^X]d_ _Vda _d _UZ YXX]V`Y_Vda duZ[Vaj _UZ WZ[\V`Z ^WVaj W`[VX_W Y[Z X[d\VbZb cd[ \VW^Y]VrVaj _UZ d^_X^_ YW kZ]] ghi ˆd ff `daaZ`_n l

   ! " #$%&' TUZ m[W_ ]Vs[Y[q kUV`U kZ kd^]b ]V‰Z _d YaY]qrZ [Ze jY[bVaj V_W XZ[cd[ f Ya`Z Yab ^WYsV]V_q dc _UVW XY[_ dc _UZ ghiel WXZ`Vm`Y_Vdap VW _UZ Vf X]Z f Za_Y_Vda c[d f   ˆn vZ YaY]qrZb _UZ XZ[cd[ f Ya`Z dc _UZ ]Vs[Y[q da Y WVaj]Z adbZ Va_[YeadbZ `d ff ^aV`Y_Vda  YW kZ]] YW sZ_kZZa X[d`ZWWZW da WZXY[Y_Z adbZW ^WVaj _UZ   ˆ i ¸ WkV_`U Va_Z[eadbZ `d ff ^aV`Y_Vda  n ¸_Y[_Vaj Ya YXX]V`Y_Vda kUV`U VW jdVaj _d ^WZ ghi ˆd ff WXYkap _UZ ^WZ[ UYW _d WXZ`Vcq Ya YbbVe _VdaY] XY[Y f Z_Z[ `Y]]Zb ()*+,- p kUV`U VabV`Y_ZW _UZ f YVf ^ f a^ f sZ[ dc X[d`ZWWZW ^WZb kV_UVa _UZ ]VcZe _Vf Z dc _UZ YXX]V`Y_Vdan od[ ZY f X]Zp Vc _UZ YXX]V`Ye _Vda VW W_Y[_Zb d[VjVaY]]q kV_U ‹ X[d`ZWWZW Yab _UZ ^WZ[ ]Y_Z[ kYa_W _d WXYka da cd^[ YbbV_VdaY] X[d`ZWWZWp _UZ `d ff Yab ]VaZ kd^]b aZZb _d sZ YW cd]]dkW³ (-./0, 1,- 2 1()*+,- 3 456(7)--8 vUV]Z _UVW YXX[dY`U VW ZX]V`V_]q Y]]dkZb sq _UZ ghie l WXZ`Vm`Y_Vdap V_ Y]Wd `]ZY[]q WZ_W `Z[_YVa ]Vf V_W da _UZ bqaY f V` sZUY\Vd[ dc _UZ YXX]V`Y_Vdan vUZa ^WVaj _UZ `daaZ`_ƒY``ZX_ YXX[dY`Up _UZ ^WZ[ UYW _d WZ_ Yad_UZ[ ¹Yj cd[ `d f XV]Vaj Yab W_Y[_e Vaj _UZ YXX]V`Y_Vdan TUZ 9:-.- ¹Yj W_[daj]q Vae bV`Y_ZW Y][ZYbqp _UY_ _UZ ghi ˆd ff `daaZ`_ Yab ghi ˆd ff Y``ZX_ c^a`_VdaW UY\Z sZZa Vf X]Z f Za_Zb Va ghiƒ¸ ^WVaj Tˆhƒihn „a Va_Z[ZW_Vaj ² ^ZW_Vda [ZjY[bVaj _UVW ¹Yj VWp kUZ_UZ[ `d ff ^aV`Y_Vda Va ZY`U dc _UZ VabZXZabZa_p XY[Y]]Z] YXX]V`Y_VdaW VW Va¹^Za`Zb sq _UVW ¹Yjp Znjn kUZ_UZ[ Y]] `d ff ^aV`Y_Vda VW Ze Z`^_Zb ^WVaj Tˆhƒihp d[ kUZ_UZ[ ‚^W_ _UZ `d ff ^e aV`Y_Vda sZ_kZZa _UZ _kd YXX]V`Y_VdaW `daaZ`_Zb sq _UZ bqaY f V`Y]]q `[ZY_Zb Va_Z[e`d ff ^aV`Y_d[ VW ^WVaj Tˆhƒihn oVj^[Z t WUdkW _UZ f YVf ^ f sYabkVb_U Y`UVZ\Zb kV_U _UZ bVuZ[Za_ `d ff ^aV`Y_d[W ^WVaj Y XVajeXdaj sZa`U f Y[‰ sZ_kZZa _kd X[d`ZWWZWn ‡s\Vd^W]qp _UZ XZ[cd[ f Ya`Z Y`UVZ\Zb kV_U Y `d ff ^aV`Y_d[ `[ZY_Zb sq ghi ˆd ff WXYka VW VbZa_V`Y] _d _UZ W_Y_V` YXe X[dY`Un  dkZ\Z[p ^WVaj _UZ `d ff ^aV`Y_d[ `[ZY_Zb sq ghi ˆd ff `daaZ`_ƒY``ZX_ YXX[dY`U XZ[cd[ f W WVje aVm`Ya_]q kd[WZp WVa`Z _UZ Tˆhƒih XZ[cd[ f Ya`Z dc _UZ f Y`UVaZ `Ya ad_ `d f XZ_Z kV_U _UZ sYabkVb_U Y`UVZ\Zb _U[d^jU _UZ [Zj^]Y[ `d ff ^aV`Y_Vda bZ\V`Zn

n ˆdaaZ`_ _kd Y][ZYbq [^aaVaj YXX]V`Y_Vda X[de `ZWWZWp kUV`U UY\Z Y Wd`‰Z_ `daaZ`_Vda ZW_Yse ]VWUZb sq ^WVaj ghi ˆd ff ‚dVan TUZ [ZW^]_Vaj `d ff ^aV`Y_d[ ^WZW _UZ X[ZeZW_Ys]VWUZb Wd`‰Z_ `daaZ`_Vda cd[ _UZ bY_Y Z`UYajZ Yab VW [Ze W_[V`_Zb _d _UZ X[d`ZWWZW Ud]bVaj _UZ Wd`‰Z_e `daaZ`_Vdan ia _UZ c[Y f Z dc _UVW XYXZ[ kZ kd^]b ]V‰Z _d cd`^W da _UZ m[W_ _kd f Z_UdbWn TUZ ZZe `^_Zb _ZW_W `d f XY[Z _UZ sYabkVb_U Yab ]Y_Za`q Y`UVZ\Zb ^WVaj bqaY f V` `d ff ^aV`Y_d[W `[ZY_Zb sq ghi ˆd ff WXYkap Ya Va_Z[e`d ff ^aV`Y_d[ [ZW^]_Vaj c[d f _UZ `d^X]Vaj dc _kd VabZXZabZa_ YXX]V`Y_VdaW ^We Vaj ghi ˆd ff `daaZ`_ Yab ghi ˆd ff Y``ZX_ Yab Y W_Y_V` `d ff ^aV`Y_d[ _qXV`Y] cd[ ghietn vZ c^[e _UZ[ f d[Z Va\ZW_VjY_Z _UZ `dW_W cd[ WXYkaVaj X[d`ZWWZWp WVa`Z f Yaq bqaY f V` W`ZaY[VdW kd^]b sZaZm_ c[d f Y ¹ZVs]Z UYab]Vaj dc `UV]b X[d`ZWWZWn  d_ ZY f VaZb Va _UVW Va\ZW_VjY_Vda VW Udk bVuZ[e Za_ sY_`UeW`UZb^]Z[W `d^]b UYab]Z YXX]V`Y_VdaWp kUV`U bqaY f V`Y]]q `UYajZ _UZ a^ f sZ[ dc X[d`ZWWZW b^[Vaj [^a_Vf Zn vUV]Z _UVW VW Ya Va_Z[ZW_Vaj Yab Vf Xd[_Ya_ ² ^ZW_Vdap V_ VW ad_ Y X[dXZ[_q dc _UZ ghi ]Vs[Y[qp Yab bZXZabW W_[daj]q da _UZ ]d`Y] VaW_Y]]Y_Vda XY[Y f Z_Z[W dc _UZ sY_`UeW`UZb^]Z[ da _UZ f Y`UVaZWn TUZ ghi ]Vs[Y[VZW ZY f VaZb Va _UVW WZ`_Vda Y[Z³
 ³ ]Vs[Y[q \Z[WVda  n†nln TZW_W kZ[Z Ze Z`^_Zb da Ya   ˆ ¸ e  `daWVW_Vaj dc  adbZWp ZY`U UY\Vaj  †Š g  r X[d`ZWWd[W kV_U  ‹ q_ZW dc f YVa f Z f d[q XZ[ adbZn   
 ³ ]Vs[Y[q \Z[WVda nŠ†n TZW_W kZ[Z ZZ`^_Zb da Y  V_Y`UV ¸´  ŠŠŠ kV_U t adbZWp ZY`U UY\Vaj  l Š g  r X[d`ZWWd[Wn  Y`U adbZ UYW  q_ZW dc f YVa f Z f d[qn 
 ³ ]Vs[Y[q \Z[WVda  n TZW_W kZ[Z Ze Z`^_Zb da Y ¸  oV[Z  ŠŠp kV_U l‹ † Š g  r ¸h„´ˆ iii X[d`ZWWd[Wp Yab …  q_ZW dc f YVa f Z f d[qn  
 ³ ]Vs[Y[q \Z[WVda †nŠn‹n TZW_W kZ[Z ZZ`^_Zb da Y `]^W_Z[ kV_U l adbZWp ZY`U UY\Vaj _kd ln‹  r hZa_V^ f ‹ Zda X[d`ZWWd[W Yab l q_ZW dc f Z f d[qn TUZ adbZW Y[Z `daaZ`_Zb sq VjYsV_  _UZ[aZ_n 

Point-to-point performance with various communicators

static intra-node spawn intra-node connect/accept intra-node static inter-node spawn inter-node connect/accept inter-node

1400

1200 Maximum Bandwidth [MBytes/sec]

100000 Maxmimum Bandwidth [MBytes/sec]

Point-to-point performance Hitachi SR8000 intra-node

static-intra spawn-intra connect/accept-intra static-inter spawn-inter connect/accept-inter

10000

1000

100

1000

800

600

400

200 10 0 0

0.5

1 Message Length [MBytes]

1.5

2

0

0.5

1 Message Length [MBytes]

1.5

2

oVj^[Z t³ hdVa_ _d XdVa_ XZ[cd[ f Ya`Z da _UZ ¸ e  cd[ oVj^[Z l³ hdVa_ _d XdVa_ XZ[cd[ f Ya`Z da _UZ  V_Y`UV ¸´  ŠŠŠ cd[ \Y[Vd^W `d ff ^aV`Y_d[W \Y[Vd^W `d ff ^aV`Y_d[W

   ! &    " #$ TUZ [ZW^]_W Y`UVZ\Zb kV_U ¸  eghi Y[Z X[ZWZa_Zb Va mj^[Z n Td W^ ff Y[VrZ _UZWZ [ZW^]_W Yab _UZ ZXZ[Ve Za`ZWp ad YbbV_VdaY] ¹YjW UYb _d sZ ^WZb _d f Y‰Z Yaq dc _UZ ZY f X]ZW kd[‰p Yab _UZ XZ[cd[ f Ya`Z Y`UVZ\Zb Va Y]] W`ZaY[VdW _ZW_Zb kZ[Z Y]kYqW sYWV`Y]]q VbZa_V`Y] _d _UZ W_Y_V` ghi ˆ‡gg v‡´µ¶ W`ZaY[Vdn    !  " %" #$  

WVaj µ„gƒghi \n†nŠn‹p Y]] _U[ZZ _ZW_W X[d\VbZb sYe WV`Y]]q _UZ WY f Z XZ[cd[ f Ya`Zp YW WUdka Va mj^[Z ‹n vZ `dab^`_Zb YjYVa _kd WZ_W dc _ZW_Wp daZ ^WVaj _kd X[d`ZWWZW da WZXY[Y_Z adbZW Va_Z[eadbZ _ZW_W  Yab daZ ^WVaj _kd X[d`ZWWZW da Y WVaj]Z adbZ Va_[YeadbZ _ZW_W  n od[ _UZ Va_[YeadbZ _ZW_Wp kZ ^WZb _UZ 7  WUY[Zb f Z f d[q f db^]Zn vZ kd^]b ]V‰Z _d `d ff Za_ UdkZ\Z[ da _UZ sZUY\Vd[ dc µ„gƒghi kUZa ^WVaj ghi ˆd ff WXYkan vUZa sdd_Vaj _UZ ]Y f eUdW_Wp _UZ ^WZ[ UYW _d WXZ`Vcq Va Y UdW_m]Z _UZ ]VW_ dc f Y`UVaZWp kUV`U WUd^]b sZ ^WZb cd[ _UZ XY[Y]]Z] ‚dsn „ µ„g bYZ f da VW _UZa W_Y[_Zb da ZY`U dc _UZWZ f Y`UVaZWn TUZ X[d`ZWWZW dc Y XY[e Y]]Z] ‚ds Y[Z W_Y[_Zb Y``d[bVaj _d _UZV[ d[bZ[ Va _UZ UdW_m]Zn vUZa `Y]]Vaj ghi ˆd ff WXYkap _UZ aZk X[d`ZWWZW Y[Z W_Y[_Zb YjYVa ^WVaj _UZ m[W_ f Y`UVaZ Va _UZ ]VW_p Vc ad UVa_W Y[Z jV\Za _d _UZ WqW_Z f ^We

o^[_UZ[ Va\ZW_VjY_VdaW WUdkZbp _UY_ _UZ 9:-.- ¹Yj bdZW ad_ WZZ f _d Va¹^Za`Z _UZ f YVf ^ f Y`UVZ\e Ys]Z sYabkVb_U kV_UVa ZY`U VabZXZabZa_ YXX]V`Y_Vdan  dkZ\Z[p d^[ f ZYW^[Z f Za_W WUdkZbp _UY_ _UZ \Y[Ve Ya`Z kYW W]VjU_]q UVjUZ[ _UYa _UZ _ZW_ `YWZ ad_ ^WVaj _UZ 9:-.- ¹Yjn TUZ W_YabY[b bZ\VY_Vda kV_Ud^_ _UZ 9:-.- ¹Yj kYW ^W^Y]]q sZ]dk t p kUV]Z ^WVaj _UZ 9:-.- ¹Yj V_ kYW Va _UZ [YajZ dc etŠ n TUVW f VjU_ sZ _UZ [ZW^]_ dc d``YWVdaY] Xd]]Vaj dc Tˆhƒih Wd`‰Z_Wn


  !     " #$ „W da _UZ X[Z\Vd^W f Y`UVaZp kZ `dab^`_Zb _kd WZ_W cd[ ZY`U ZXZ[Vf Za_ da _UZ  V_Y`UV ¸´  ŠŠŠ³ Y]] _ZW_W kZ[Z ZZ`^_Zb ^WVaj _kd X[d`ZWWZW da _UZ WY f Z adbZp VabV`Y_Zb Va mj^[Z l YW Va_[YeadbZ `d ff ^aV`Y_Vdap Yab ^WVaj _kd X[d`ZWWZW da bVuZ[e Za_ adbZWp [ZcZ[[Zb _d YW Va_Z[eadbZ `d ff ^aV`Ye _Vdan „W WUdka Va mj^[Z l _UZ XZ[cd[ f Ya`Z Y`UVZ\Zb kV_U _UZ ghi ˆd ff WXYka ZY f X]Z Yab kV_U _UZ ghi ˆd ff `daaZ`_ƒY``ZX_ ZY f X]Z VW `d f XY[Ys]Z _d _UZ W_Y_V` YXX[dY`U cd[ _UZ Va_Z[eadbZ _ZW_Wn od[ _UZ Va_[YeadbZ _ZW_ `YWZWp Y]] _ZW_W Y[Z Y`UVZ\Vaj _UZ WY f Z sYabkVb_U cd[ ]Y[jZ f ZWWYjZWn od[ Wf Y]]Z[ f ZWe WYjZWp _UZ d\Z[Y]] XZ[cd[ f Ya`Z VW X[dsYs]q VbZa_V`Y] YW kZ]]p Z\Za Vc f Vad[ \Y[VY_VdaW Y[Z dsWZ[\Ys]Z b^Z _d `Y`UVaj ZuZ`_Wn ‹

Point-to-point performance SUN Fire 6800

Point-to-point performance with LAM-7.0.4 using various communicators

1000

500 static spawn connect/accept

900

400 Maxmimum Bandwidth [MBytes/sec]

800 Maximum Bandwidth [MBytes/sec]

static intra-node spawn intra-node connect/accept intra-node static inter-node spawn inter-node connect/accept inter-node

450

700 600 500 400 300

350 300 250 200 150

200

100

100

50 0

0 0

0.5

1 Message Length [MBytes]

1.5

2

0

0.5

1 Message Length [MBytes]

1.5

2

oVj^[Z ³ hdVa_ _d XdVa_ XZ[cd[ f Ya`Z da _UZ ¸  e oVj^[Z ‹³ hdVa_ _d XdVa_ XZ[cd[ f Ya`Z kV_U µ„gƒghi oV[Z cd[ \Y[Vd^W `d ff ^aV`Y_d[W cd[ \Y[Vd^W `d ff ^aV`Y_d[W Vaj _UZ Y``d[bVaj ghi iacd ds‚Z`_n ibZY]]qp _UZ ^WZ[ kd^]b ZXZ`_p _UY_ _UZ m[W_ ^a^WZb adbZ Y_ ]ZYW_ ^a^WZb Y``d[bVaj _d _UZ ‚ds kUV`U WXYkaW _UZ X[de `ZWWZW  VW `UdWZap _d bVW_[Vs^_Z _UZ ]dYb YXX[dX[VY_Z]qn vV_U _UZ `^[[Za_ W`UZ f Zp V_ VW X[dsYs]Z _UY_ cd[ `d f e X^_Z Va_ZaWV\Z YXX]V`Y_VdaW _UZ d\Z[Y]] ‚ds kV]] W]dk bdka sq WXYkaVaj YbbV_VdaY] X[d`ZWWZW da _UZ adbZW kUV`U Y[Z Y][ZYbq [^aaVaj Ya ghie‚dsn TUZ ^WZ[ UYW _UZ XdWWVsV]V_q _d `UYajZ _UVW sZUY\Vd[ sq ^WVaj WXZe `Vm` ghi iacd ds‚Z`_W kUZa WXYkaVaj aZk X[d`ZWWn )(+ -)
,+  :  +/  0,  +/  ., Va_[db^`ZW Y [d^abe [dsVa W`UZb^]Vaj dc X[d`ZWWZW jV\Vaj _UZ YXX]V`Y_Vda _UZ XdWWVsV]V_q _d WXZ`Vcq _UZ m[W_ UdW_ kUV`U kd^]b sZ ^WZbn )(+,  +/   9+,   +  :  0  Z`]^bZW _UZ [dd_ adbZ dc _UZ WXYkae`d ff Yab kUZa `[ZY_Vaj aZk X[d`ZWWZWn TUVW WZ`dab f dbZ VW Va_ZabZb cd[ W^Xe Xd[_Vaj XY[Y]]Z] YXX]V`Y_VdaW ^WVaj Y f YaYjZ[ekd[‰Z[ XY[YbVj f n

   " #$      ¸d f Z YXX]V`Y_VdaW f VjU_ sZaZm_ c[d f Y c[Z ² ^Za_]q `UYajVaj a^ f sZ[ dc X[d`ZWWZWp Znjn Wd f Z ghi YXe X]V`Y_Vda Vf X]Z f Za_Vaj Wd f Z _qXZ dc `d f X^_Y_VdaY] WZ[\V`Zp \Y[Vd^W f YaYjZ[ekd[‰Z[ W`ZaY[VdW d[ WVf ^]Ye _VdaW ^WVaj YbYX_V\Z f ZWU [ZmaZ f Za_n TUVW WZ`_Vda W^ ff Y[VrZW _UZ `dW_W dc WXYkaVaj daZ _d cd^[ X[de

`ZWWZW da ZY`U dc _UZ YaY]qrZb f Y`UVaZWn „]] dc _UZWZ _ZW_W UY\Z sZZa ZZ`^_Zb sq [^aaVaj _UZ `dbZ Va_Z[Y`e _V\]qp WVa`Z WXYkaVaj dc X[d`ZWWZW Va Y sY_`U Za\V[dae f Za_ VW da f dW_ X]Y_cd[ f W ad_ W^XXd[_Zbn  \Za da f Y`UVaZW W^XXd[_Vaj _UVW c^a`_VdaY]V_qp _UZ WXYkaVaj f VjU_ _UZa Va`]^bZ Ya Y[sV_[Y[q _Vf ZW]V`Z cd[ Y]]d`Y_e Vaj X[d`ZWWd[W _U[d^jU _UZ W`UZb^]Z[n vZ `d^]b dsWZ[\Z da Y]] X]Y_cd[ f Wp _UY_ _UZ `dW_W cd[ WXYkaVaj X[d`ZWWZW kYW Y_ ]ZYW_ Ya d[bZ[ dc f Yje aV_^bZ UVjUZ[ _UYa _UZ XdVa_e_deXdVa_ ]Y_Za`qn gdW_ ad_Ys]qp _UZ `dW_W cd[ WXYkaVaj X[d`ZWWZW da _UZ ¸^a oV[Z Y[Z Y[d^ab ‹Š f V]]VWZ`dabW Yab da _UZ  V_Y`UV ¸´  ŠŠŠ UVjUZ[ _UYa ‹ŠŠ f V]]VWZ`dabWn vZ Y[Z ad_ bVW_Vaj^VWUVaj UZ[Z sZ_kZZa _UZ Va_Z[e Yab _UZ Va_[Ye adbZ `YWZp WVa`Z _UZ f Y`UVaZ Y]kYqW WXYkaZb _UZ m[W_ X[d`ZWW da _UZ adbZ kUZ[Z _UZ XY[Za_ X[d`ZWW UYW sZZa [^aaVajp Yab _UZ cd]]dkeda X[d`ZWWZW da d_UZ[ adbZWn TU^Wp kZ Y]kYqW UYb Y f VZb Va_[Ye Yab Va_Z[e adbZ Za\V[da f Za_n od[ ghiƒ¸ Yab µ„gƒghip _UZ `dW_W cd[ WXYkae Vaj Va`[ZYWZW W]VjU_]q kV_U _UZ a^ f sZ[ dc aZk X[de `ZWWZWn  dkZ\Z[p _UZ d\Z[Y]] `dW_W kZ[Z Va Y ]dk f V]e ]VWZ`dab Y[ZYn

 dn dc t l  ‹ X[d`W WXYkaZb ¸ e  Va_[Y ‹nt ‹n… n…  n† ¸ e  Va_Z[ t n† t…n… t…n t…n… ¸´  ‹Š… ‹t… ‹‹t ‹  t ¸    n ‹Šn ‹n† ‹Šn‹ µ„g Va_[Y Šn tn tn ln‹ µ„g Va_Z[ Šn… tn tn† lnŠ TYs]Z t³  Z`^_Vda _Vf Z cd[ WXYkaVaj \Y[Vd^W a^ f sZ[ dc X[d`ZWWZW Va f V]]VWZ`dabWn

Q zQ  R y S  z  z y   O}{  { z   „ad_UZ[ `Y]] c[d f _UZ ghiel bd`^ f Za_ VW dc XY[e _V`^]Y[ Va_Z[ZW_ cd[ _UZ daZeWVbZb dXZ[Y_VdaWp aY f Z]q  QRP}z y O _UZ XdWWVsV]V_q _d Y]]d`Y_Z Wd f Z cYW_  f Z f d[q ^We Vaj ghi „]]d` f Z f ~€n ‡a WUY[Zb f Z f d[q Y[`UVe TUZ `UYX_Z[ Ysd^_ daZeWVbZb `d ff ^aV`Y_Vda kYW _Z`_^[ZW f VjU_ sZ cd[ ZY f X]Z Y WUY[Zb f Z f d[q d[VjVaY]]q X]YaaZb _d sZ _UZ f dW_ b[Y f Y_V` W^XX]Ze WZj f Za_ _UVW kUV`U `Ya sZ bV[Z`_]q Y``ZWWZb sq Y j[d^X dc f Za_ _d _UZ ghiet WXZ`Vm`Y_Vdap WVa`Z V_ WXZ`VmZW X[d`ZWWZWn TUZ[Zcd[Zp ´g„ dXZ[Y_VdaW Yab daZeWVbZb Y `d f X]Z_Z]q aZk XY[YbVj f cd[ Z`UYajVaj bY_Y sZe `d ff ^aV`Y_Vda f VjU_ sZ cYW_Z[p Vc f Z f d[q Y[ZYW Y[Z _kZZa X[d`ZWWZWn ia `da_[Y[q _d _UZ _kdeWVbZb `d f e Va\d]\Zbp kUV`U UY\Z sZZa Y]]d`Y_Zb \VY _UVW c^a`_Vdan f ^aV`Y_Vda dc ghietp Y WVaj]Z X[d`ZWW `da_[d]W _UZ XYe „ f daj _UZ YaY]qrZb ghi ]Vs[Y[VZW Y[Z³ [Y f Z_Z[W cd[ Wd^[`Z Yab bZW_VaY_Vda X[d`ZWWZWn  dke Z\Z[p WVa`Z _UZ jdY] kYW _d bZWVja Y Xd[_Ys]Z Va_Z[cY`Z cd[ daZeWVbZb dXZ[Y_VdaWp _UZ WXZ`Vm`Y_Vda UYW sZ`d f Z
 ³ ]Vs[Y[q \Z[WVda  n†nln TZW_W kZ[Z ZZe `^_Zb da Ya   ˆ ¸ e  `daWVW_Vaj dc ZVjU_ adbZW [Y_UZ[ `d f X]Zn i_ `Ya sZ s[VZ¹q W^ ff Y[VrZb YW cd]e kV_U ZVjU_ †Šg  r X[d`ZWWd[W ZY`Un ]dkW³


 





Td f d\Z bY_Y c[d f _UZ f Z f d[q dc daZ X[d`ZWW _d _UZ f Z f d[q dc Yad_UZ[ X[d`ZWWZWp _U[ZZ dXe Z[Y_VdaW Y[Z X[d\VbZb³ ghi Z_p ghi h^_ Yab ghi „``^ f ^]Y_Zp _UZ ]Y__Z[ daZ `d f sVaVaj _UZ bY_Y dc _UZ _Y[jZ_ X[d`ZWWZW Va Y WVf V]Y[ cYWUVda _d ghi ´Zb^`Zn





ghiel bZmaZW c^[_UZ[ f d[Z _U[ZZ bVuZ[Za_ f Z_UdbW _d Wqa`U[daVrZ _UZ X[d`ZWWZW Vae \d]\Zb Va _UZ bY_Y Z`UYajZ³ ghi vVa cZa`Zp ghi vVa W_Y[_ƒXdW_ƒkYV_ƒ`d f X]Z_Z Yab ghi vVa ]d`‰ƒ^a]d`‰n TUZ m[W_ _kd f Z_Ue dbW Y[Z `Y]]Zb        p WVa`Z _UZ bZW_VaY_Vda X[d`ZWWZW VW Y]Wd Va\d]\Zb Va _UZ dXZ[Y_Vdan TUZ ]YW_ f Z_Udb VW `Y]]Zb         p WVa`Z _UZ _Y[jZ_ X[d`ZWW VW ad_ XY[_V`VXY_Vaj Va Yaq dc _UZ ghie`Y]]Wn







  
 ³ ]Vs[Y[q \Z[WVda nŠ†n TZW_W kZ[Z ZZ`^_Zb da _UZ WY f Z f Y`UVaZ ]V‰Z Va _UZ X[Z\Ve d^W WZ`_Vdan   
 ³ XXZnXdZ \Z[WVda ‹ntnŠn‹n TZW_W kZ[Z ZZ`^_Zb da Ya i g X  …Šp ZY`U adbZ `daWVW_Vaj dc l tn†  r hdkZ[‹  X[d`ZWWd[W kV_U tl 

f Z f d[q XZ[ adbZp „i \Z[WVda nln 
 ³ ]Vs[Y[q \Z[WVda  n TZW_W kZ[Z ZZe `^_Zb da _UZ WY f Z f Y`UVaZ YW bZW`[VsZb X[Z\Ve d^W]q Va WZ`_Vda nn  
 ³ ]Vs[Y[q \Z[WVda †nŠn‹n TZW_W kZ[Z ZZ`^_Zb da _UZ WY f Z `]^W_Z[ YW ^WZb cd[ _UZ _ZW_W Va _UZ X[Z\Vd^W WZ`_Vdan





  
 

!


 _[YaWcZ[[Vaj _UZ bY_Y sY`‰n TVf Vaj VW bdaZ Va`]^bVaj _UZ d\Z[Y]] ZZ`^_Vda _Vf Z cd[ sd_U dXZ[Y_VdaW TUZ   cd]]dkVaj `dbZ c[Yj f Za_ VW VabV`Y_Vaj _UZ sZa`U f Y[‰ ‡aZeWVbZb dXZ[Y_VdaW Y[Z f ZYa_ _d sZ ^WZb Va WV_^e `dbZ cd[ _UZ _ZW_e`YWZ ^WVaj ghi vVa cZa`Z cd[ Wqae Y_VdaWp kUZ[Z daZ X[d`ZWW VW Va `da_[d] dc _UZ WZabZ[ `U[daVrY_Vdan Yab [Z`ZV\Z[ WVbZ XY[Y f Z_Z[Wp Znjn _UZ [Z`ZV\Z[ X[d`ZWW bdZW ad_ ‰adk _UZ X[Z`VWZ WVrZ d[ _UZ X[Z`VWZ a^ f sZ[   

dc Va`d f Vaj f ZWWYjZWn TqXV`Y] ^WYjZ W`ZaY[VdW f VjU_ . 5  /), ):9.   -)/9 ., 9   -., -  ,  5 Va\d]\Z cd[ ZY f X]Z bY_Y Z`UYajZ sZ_kZZa bVuZ[Za_  + .,+   ,:    
.,  bd f YVaW Va YXX]V`Y_VdaW ^WVaj ^aW_[^`_^[Zb f ZWUZWn + 09   )9)97-   444 
., 

  TUZ sYWV` WZ ² ^Za`Z dc c^a`_VdaW kUZa ^WVaj Y`_V\Z  + .,+ ,:0   :,9  
.,  _Y[jZ_ Wqa`U[daVrY_Vda VW YW cd]]dkW³ X[d`ZWWZW `Ya Y]]dk Y``ZWW _d Y `Z[_YVa Y[ZY Va _UZV[ f Z f d[q _d 5  -)  .  -)/9 ., 9   -., -  ,  5 d_UZ[ X[d`ZWWZWp VaV_VY_Vaj Y Wde`Y]]Zb ZXdW^[Z ZXd`Un    ,:    
.,  „ X[d`ZWW f ^W_ c^[_UZ[ f d[Z dXZa Ya Y``ZWW ZXd`Up  ++ .,+ .,+   ,:    
.,  Vc UZ kd^]b ]V‰Z _d X^_ bY_Y Va_d d[ jZ_ bY_Y c[d f  _UZ f Z f d[q dc Yad_UZ[ X[d`ZWWn ghiel bdZW UdkZ\Z[    .  /),   

ad_ j^Y[Ya_ZZp _UY_ _UZ bY_Y _[YaWcZ[ VW maVWUZb Yce 5  -)  .  -)/9 ., 9   -., -  ,  5 _Z[ ghi h^_ d[ ghi Z_ UY\Z [Z_^[aZbp _UZ [ZW^]_ dc    ,:    
.,  _UZ bY_Y _[YaWcZ[ f VjU_ da]q sZ \VWVs]Z kUZa `]dWVaj  ++ .,+ .,+   ,:    
.,  _UZ Y``ZWW Yab ZXdW^[Z ZXd`Un Z`Y^WZ dc _UVW X[dXe Z[_qp Y `dbZ kV]] _qXV`Y]]q VaV_VY_Z _UZ bY_Y _[YaWcZ[ 5  ):9.  -)/9 ., 9   -., -  ,  5 ^WVaj daZeWVbZb dXZ[Y_VdaW Yab `]dWZ _UZ Y``ZWW Yab  + .,+   ,:   
.,  ZXdW^[Z ZXd`U Y_ _UZ XdVa_p kUZ[Z _UZ bY_Y _[YaWe +  09   0   :,9   )9)97-   444 
.,  cZ[ UYW _d sZ maVWUZb c[d f _UZ YXX]V`Y_Vda XdVa_ dc  + .,+   ,:   
., 

 \VZkn TUVW sZUY\Vd[ VW WVf V]Y[ _d adaes]d`‰Vaj _kde  WVbZb `d ff ^aV`Y_Vda Va ghietn „W Y [ZW^]_ dc _UVW X[dXZ[_qp Va ZY`U Y``ZWW Yab Ze vUV]Z Y XVajeXdaj sZa`U f Y[‰ VW ad_ aZ`ZWWY[V]q Y XdW^[Z ZXd`U Ya YXX]V`Y_Vda kV]] ZZ`^_Z Y ]Vf V_Zb _qXV`Y] `d ff ^aV`Y_Vda XY__Z[a kUZa ^WVaj daZeWVbZb a^ f sZ[ dc daZeWVbZb dXZ[Y_VdaW sZcd[Z maVWUVaj Ya dXZ[Y_VdaWp kZ cd^ab _UY_ V_ W_V]] [Z\ZY]W _UZ [Z]Z\Ya_ Y``ZWW Yab ZXdW^[Z ZXd`U Yab ]Y^a`UVaj aZk daZWn `d ff ^aV`Y_Vda f Z_Z[W dc daZeWVbZb dXZ[Y_VdaWn TUZ sYWV` XZ[cd[ f Ya`Z `UY[Y`_Z[VW_V`W dc daZeWVbZb Td c^[_UZ[ ‚^bjZXY[Y _UZ ghi Vf X]Z f Za_Y_VdaWp kZ `dae dXZ[Y_VdaW `d^]b _UZ[Zcd[Z sZ f ZYW^[Zb Va bZ_Z[ f Vae b^`_Zb Wd f Z YbbV_VdaY] _ZW_W ^WVaj W]VjU_ f dbVm`Ye Vaj _UZ Wqa`U[daVrY_Vda `dW_Wp VnZn _UZ `dW_W _d dXZa _VdaW dc _UZ XVajeXdaj sZa`U f Y[‰ bZW`[VsZb Ysd\Z³ Yab _d `]dWZ Y``ZWW Yab ZXdW^[Z ZXd`UWp YW kZ]] YW _UZ Y`UVZ\Ys]Z sYabkVb_U ^WVaj daZeWVbZb dXZ[Y_VdaWn „aY]qrZb _UZ sZUY\Vd[ dc daZeWVbZb dXZ[Y_VdaW TUZWZ _kd XY[Y f Z_Z[W Y[Z _UZ cd`^W dc _UVW XYXZ[p Yab ^WVaj bZ[V\Zb bY_Y_qXZW kV]] sZ f ZYW^[Zb sq ZZ`^_Vaj Y XVajeXdaj _ZW_ ^WVaj daZeWVbZb dXZ[Y_VdaWn „aY]qrZb _UZ XZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Ye Td Vf X]Z f Za_ Y XVajeXdaj sZa`U f Y[‰ ^WVaj daZe _VdaW cd[ f ^]_VX]Z X^_ƒjZ_ dXZ[Y_VdaW sZ_kZZa WVbZb dXZ[Y_VdaWp ZY`U X[d`ZWW `[ZY_ZW m[W_ Ya Y``ZWW _UZ WY f Z XYV[ dc X[d`ZWWZW Yab ZXdW^[Z ZXd`U Yab X^_W d[ jZ_W bY_Y Vaƒc[d f _UZ [Z f d_Z f Z f d[q ^WVaj Y WVaj]Z ghi dXZ[Y_Vdan „c_Z[ „aY]qrZb kUZ_UZ[ _UZ X[d`ZWWZW ad_ Y`_V\Z]q Vae `]dWVaj _UZ Y``ZWW Yab ZXdW^[Z ZXd`U da sd_U X[de \d]\Zb Va _UZ daZeWVbZb dXZ[Y_Vda `Ya ZZ`^_Z `ZWWZW Yab _U^W cd[`Vaj Y]] dXZ[Y_VdaW _d maVWUp sd_U `Y]`^]Y_VdaWp kV_Ud^_ YuZ`_Vaj _UZ XZ[cd[ f Ya`Z X[d`ZWWZW `[ZY_Z Y WZ`dab ZXdW^[Z Yab Y``ZWW ZXd`Up dc _UZ bY_Y _[YaWcZ[n

     



†

¸Zabƒ´Z`\ vVa cZa`Z vVa cZa`Z  W_Y[_ƒXdW_ W_Y[_ƒXdW_  ghi „]]d` f Z f ghi „]]d` f Z f ¸ e  Va_[Y ‹lnl l  n… nŠ n n ¸ e  Va_Z[ ‹lnl l  n… nŠ n n ¸´  Va_[Y ttnl  ‹n… e t ln… e ¸´  Va_Z[ lln tt…n e l  n‹ e ¸  ln ‹n† l…n‹  n n i g Va_[Y  n… e  …nl e n i g Va_Z[ ttn tŠ‹nŠ e †  n‹ e µ„g Va_[Y t n‹ t ‹n e  ‹nt e µ„g Va_Z[ ‹‹n† l †n e tl†nl e TYs]Z l³  Z`^_Vda _Vf Z cd[ _[YaWcZ[[Vaj Y cd^[esq_Z f ZWWYjZ ^WVaj bVuZ[Za_ `d ff ^aV`Y_Vda f Z_UdbW Va Wn ghiel Y]]dkW dX_Vf VrY_VdaW dc daZeWVbZb dXZ[Y_VdaW da Y j[d^X ]Z\Z]p kUV`U kd^]b [Z ² ^V[Z YbbV_VdaY] XYe [Y f Z_Z[W _d `UY[Y`_Z[VrZ _UZ f n TUZ YaY]qWVW dc _UZ sZUY\Vd[ dc daZeWVbZb dXZ[Y_VdaW da Y j[d^X ]Z\Z] VW `^[[Za_]q dajdVaj kd[‰ Yab d^_WVbZ dc _UZ W`dXZ dc _UVW XYXZ[n TUZ ghiel WXZ`Vm`Y_Vda jV\ZW ^WZ[W XdWWVsV]V_VZW _d dX_Vf VrZ daZeWVbZb dXZ[Y_VdaW sq XYWWVaj UVa_W _d _UZ ghi ]Vs[Y[qp kUV`U bZW`[VsZ `Z[_YVa dX_VdaW da Udk _UZ kVabdkW Y[Z ^WZbn ¸d f Z UVa_W `Ya sZ XYWWZb YW Ya ghi iacd ds‚Z`_ kUZa `[ZY_Vaj _UZ kVabdk p kUV]Z d_UZ[W `Ya ^WZ _UZ )  /9 Y[j^ f Za_W _d _UZ Wqa`U[de aVrY_Vda [d^_VaZWn ia _UZ cd]]dkVaj _ZW_Wp _UZ bZcY^]_ \Y]^ZW ghi i  o‡  µµ Yab )  /9  UY\Z sZZa ^WZbn „a Va\ZW_VjY_Vda Va_d _UZ ZuZ`_Wp dc ZY`U dc _UZWZ XY[Y f Z_Z[W da _UZ bVuZ[Za_ WqW_Z f W kd^]b sZ \Z[q Va_Z[ZW_Vajp V_ Z`ZZbW UdkZ\Z[ _UZ W`dXZ dc _UVW XYXZ[n „bbV_VdaY]]qp _UZ ^WYjZ dc _UZWZ Y[j^ f Za_W f VjU_ dX_Vf VrZ _UZ `d ff ^aV`Y_Vda XZ[cd[ f Ya`Z da daZ X]Y_cd[ f p kUV]Z bZj[YbVaj _UZ XZ[cd[ f Ya`Z da Yad_UZ[n

WVaj XYWWV\Z _Y[jZ_ Wqa`U[daVrY_Vdap _UZ XYe [Y f Z_Z[W `UY[Y`_Z[VrVaj _UZ XZ[cd[ f Ya`Z dc _UZ dXZ[Y_Vda Y[Z bVuZ[Za_ _UYa _UZ daZW bZW`[VsZb Ysd\Zn Td f ZYW^[Z _UZ Y`UVZ\Ys]Z sYabkVb_U ^WVaj ghi vVa ]d`‰ƒ^a]d`‰p Y W_[ZY f Vaj sZa`U f Y[‰ `d^]b sZ ^WZbn od[ X[db^`Vaj `d f XY[Ys]Z [ZW^]_W kV_U [Ze WXZ`_ _d _UZ X[Z\Vd^W WZ`_Vdap kZ d f V__Zb _UZ XYWWV\Z _Y[jZ_ Wqa`U[daVrY_Vda Va _UZ cd]]dkVaj _ZW_W Yab cde `^WZb da dXZ[Y_VdaW ^WVaj Y`_V\Z _Y[jZ_ Wqa`U[daVrYe

_Vda da]qn 
&

      Td bZ_Z[ f VaZ _UZ `dW_W dc _UZ Wqa`U[daVrY_Vda dXZ[Ye _Vdap _UZ ZZ`^_Vda _Vf Z dc Y WVaj]Z cd^[esq_Z ghi h^_ dXZ[Y_Vda VW YaY]qrZbn ¸Va`Z _UZ [ZW^]_W Y`UVZ\Zb kV_U ghi Z_ Y[Z VbZa_V`Y]p _UZq UY\Z sZZa d f V__Zb Va _Ye s]Z l cd[ _UZ WY‰Z dc `]Y[V_qn TYs]Z l W^ ff Y[VrZW _UZ ZZ`^_Vda _Vf Z cd[ Y]] YaY]qrZb ghi ]Vs[Y[VZW Yab X[de \VbZW cd[ `d f XY[VWda _UZ _Vf Z _d _[YaWcZ[ Y cd^[esq_Z f ZWWYjZ ^WVaj ghi ¸Zab Yab ghi ´Z`\n „]] _Vf e VajW Y[Z jV\Za Va Wn „ f Va^W Va _UZ _Ys]Z VabV`Y_ZWp _UY_ _UZ ^WYjZ dc ghi „]]d` f Z f bVb ad_ UY\Z Yaq Va¹^Za`Z da _UZ XZ[cd[ f Ya`Z cd[ _UVW f ZWWYjZ WVrZn TUZ _d_Y] ZZ`^_Vda _Vf Z dc _UZ cd^[esq_Z _[YaWcZ[ dXZ[Y_Vda ^WVaj ghi h^_ VW cd[ Y]] YaY]qrZb ghi ]Ve s[Y[VZW WVjaVm`Ya_]q UVjUZ[ _UYa _UZ ZZ`^_Vda _Vf Z _d WZab cd^[ sq_ZW c[d f daZ X[d`ZWW _d Yad_UZ[ ^We Vaj _kdeWVbZb `d ff ^aV`Y_Vdan ¸Va`Z V_ VW \Z[q ^ae ]V‰Z]qp _UY_ _UZ bY_Y _[YaWcZ[ `dW_W cd[ _UZ cd^[ sq_ZW Y[Z _UZ `Y^WZ dc _UZ UVjU ZZ`^_Vda _Vf Z cd[ _UZ daZe WVbZb dXZ[Y_Vdap kZ `da`]^bZp _UY_ _UZ bd f VaY_Vaj XY[_ Va _UZWZ _ZW_W Y[Z _UZ `dW_W dc Wqa`U[daVrY_Vda dXe Z[Y_Vdap [ZWXZ`_V\Z]q _UZ X[d_d`d] ^WZb _d Vf X]Z f Za_ _UZ Wqa`U[daVrY_Vdan ‡a]q ¸  eghi ^WVaj f Z f d[q Y]]d`Y_Zb sq ghi „]]d` f Z f Y`UVZ\Zb Y [ZYWdaYs]Z Wf Y]]e f ZWWYjZ XZ[cd[ f Ya`Z `]dWZ _d WZabƒ[Z`\ `d f e f ^aV`Y_Vda XZ[cd[ f Ya`Zn



Two-sided vs. One-sided performance

Two-sided vs. One-sided performance

20000

8000 Send/Recv Put start/post Get start/post Put fence Get fence

7000

Maxmimum Bandwidth [MBytes/sec]

Maxmimum Bandwidth [MBytes/sec]

Send/Recv Put start/post Get start/post Put fence Get fence 15000

10000

5000

6000

5000

4000

3000

2000

1000

0

0 0

0.5

1 Message Length [MBytes]

1.5

2

0

0.5

1 Message Length [MBytes]

1.5

2

oVj^[Z ³ hZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW da _UZ   ˆ ¸ e  cd[ Va_[Y adbZ ]Zc_  Yab Va_Z[ adbZ [VjU_  `d ff ^aV`Y_Vdan Two-sided vs. One-sided performance using MPI_Alloc_mem

Two-sided vs. One-sided performance using MPI_Alloc_mem

20000

8000 Send/Recv Put-startpost Get-startpost Put-fence Get-fence

7000

Maxmimum Bandwidth [MBytes/sec]

Maxmimum Bandwidth [MBytes/sec]

Send/Recv Put-startpost Get-startpost Put-fence Get-fence 15000

10000

5000

6000

5000

4000

3000

2000

1000

0

0 0

0.5

1 Message Length [MBytes]

1.5

2

0

0.5

1 Message Length [MBytes]

1.5

2

oVj^[Z  ³ hZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW kUZa ^WVaj ghi „]]d` f Z f da _UZ   ˆ ¸ e  cd[ Va_[Y adbZ ]Zc_  Yab Va_Z[ adbZ [VjU_  `d ff ^aV`Y_Vdan Two-sided vs. One-sided performance Hitachi SR8000 intra-node

Two-sided vs. One-sided performance Hitachi SR8000 inter-node

Send/Recv Get fence Put fence Get start/post Put start/post

1400

Send/Recv Get fence Put fence Get start/post Put start/post

1000

Maxmimum Bandwidth [MBytes/sec]

Maxmimum Bandwidth [MBytes]

1200

1000

800

600

400

800

600

400

200 200

0

0 0

0.5

1 Message Length [MBytes]

1.5

2

0

0.5

1 Message Length [MBytes]

1.5

2

oVj^[Z †³ hZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW cd[ Va_[Y adbZ ]Zc_  Yab Va_Z[ adbZ [VjU_  `d ff ^aV`Y_Vda da _UZ  V_Y`UV ¸´  ŠŠŠ …



  

 
     ghi „]]d` f Z f bVb ad_ WUdk Yaq ZuZ`_ da _UZ XZ[e cd[ TUVW WZ`_Vda X[ZWZa_W _UZ sYabkVb_U Y`UVZ\Zb kV_U f Ya`Zn daZeWVbZb dXZ[Y_VdaW da Y]] YaY]qrZb X]Y_cd[ f Wn           
 TUZ [ZW^]_W Y`UVZ\Zb da Ya i g X  …Š ^WVaj Ya ZY[]q        
 dc i g W ghi ]Vs[Y[q cd[ _UZ aZk  VjU hZ[cd[e TUZ XZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW kV_U \Z[WVda ¸kV_`U Ya`Z Y[Z X[ZWZa_Zb Va mj^[Z  n ¸Vf V]Y[]q _d ghiƒ¸ kV_Ud^_ ^WVaj WXZ`VY] f Z f d[q Y]]d`Y_Zb fd_UZ[ Vf X]Z f Za_Y_VdaWp _UZ `dW_ dc _UZ Wqa`U[de sq ghi „]]d` f Z f VW ]dkZ[ _UYa [Zj^]Y[ XdVa_e_de aVrY_VdaghidXZ[Y_Vda bZ`[ZYWZ _UZ XZ[cd[ f Ya`Z dc daZe XdVa_ XZ[cd[ f Ya`Z Y`UVZ\Zb ^WVaj ghi ¸Zab Yab WVbZb dXZ[Y_VdaW kV_U i geghi cd[ Va_[YeadbZ `d f e ghi ´Z`\n TUZ YXX]V`Y_Vda `Ya W_V]] Y`UVZ\Z _UZ WY f Z f ^aV`Y_Vda kUZa XYWWVaj WUd[_ f ZWWYjZWn od[ ]Y[jZ[ f YVf ^ f sYabkVb_U kV_U daZeWVbZb dXZ[Y_VdaW YW Va f ZWWYjZ WVrZW _UZ f YVf ^ f sYabkVb_U kV_U _UZ ghiet W`ZaY[Vdp UdkZ\Z[ _UZ f ZWWYjZ WVrZ UYW _d daZ Yab _kd WVbZb `d ff ^aV`Y_Vda sZ`dY`UVZ\Zb Z WV f f V]Y[n sZ WVjaVm`Ya_]q ]Y[jZ[ _UYa cd[ _kdeWVbZb `d ff ^aV`Ye TUZ ^aZ\Za sZUY\Vd[ cd[ ]Y[jZ f ZWWYjZ WVrZW [ZX[de _Vdan TUVW VW f YVa]q b^Z _d _UZ UVjUZ[ Wqa`U[daVrYe b^`Vs]Z Yab f VjU_ sZ Y [ZW^]_ dc Udk Y]] _UZ fVWZWWYjZW _Vda `dW_W YW WUdka Va _UZ X[Z\Vd^W W^sWZ`_Vdan m_ d[ ad_  Va_d _UZ WUY[Zb tlg _UV[b ]Z\Z] `Y`UZ

WVaj ghi „]]d` f Z f _d Y]]d`Y_Z _UZ f Z f d[q WZje da ZY`U adbZ b^[Vaj _UZ _ZW_Wn f Za_W kUV`U Y[Z _UZa ^WZb Va _UZ daZeWVbZb dXZ[Ye i geghi Y`UVZ\ZW _UZ WY f Z Va_Z[eadbZ XZ[cd[e _VdaWp _UZ ^WZ[ `Ya Vf X[d\Z _UZ XZ[cd[ f Ya`Z dc daZe f Ya`Z cd[ daZ Yab _kdeWVbZb `d ff ^aV`Y_Vdan ia _UZ WVbZb dXZ[Y_VdaW cd[ sd_U _UZ ghi vVa cZa`Z Yab X[ZWZa_Zb Va mj^[Z  ad ]Y[jZ XYjZW UY\Z sZZa _UZ ghi vVa ¸_Y[_ƒhdW_ _ZW_Wn vUV]Z Va _UZ X[Ze _ZW_W `damj^[Zb da _UZ i g p kUV`U `d^]b Va`[ZYWZ _UZ XZ[e \Vd^W _ZW_ kV_Ud^_ _UZ ^WYjZ dc ghi „]]d` f Z f p cd[ f Ya`Z sZ_kZZa _UZ adbZW cd[ ZV_UZ[ f Z_Udbn _UZ ghi vVa ¸_Y[_ƒhdW_ f Z`UYaVWf kYW Y`UVZ\Vaj Y W]VjU_]q sZ__Z[ XZ[cd[ f Ya`Z _UYa _UZ ghi vVa cZa`Z       

 
 f Z`UYaVWf p _UZ bVuZ[Za`Z VW Va`[ZYWVaj WVjaVm`Ya_]q kUZa ^WVaj cYW_  f Z f d[qn TUZ [ZW^]_W Y`UVZ\Zb kV_U ¸  eghi Y[Z X[ZWZa_Zb Va mj^[Z …n Tkd f Y‚d[ ZuZ`_W `Ya sZ dsWZ[\Zbn oV[W_p _UZ ^WYjZ dc ghi „]]d` f Z f `Ya b[Y f Y_Ve           
 `Y]]q Vf X[d\Z _UZ XZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Ye TUZ [ZW^]_W cd[ _UZ  V_Y`UV Y[Z WUdka Va mj^[Z †n od[ _VdaWn ic Z f d[q VW Y]]d`Y_Zb ^WVaj _UVW c^a`_Vdap f ZWWYjZW ^X_d tn gsq_ZW Va ]Zaj_Up _UZ daZeWVbZb dXe _UZ XZ[cd[ ffYa`Z kV_U daZeWVbZb dXZ[Y_VdaW Z[Y_VdaW Y[Z ^X _d lŠ W]dkZ[ _UYa _kdeWVbZb `d ff ^e d^_XZ[cd[ f W _UZ Y`UVZ\Zb XdVa_e_deXdVa_ f Ya`Z ^WVaj aV`Y_Vdan od[ f ZWWYjZW Z`ZZbVaj _UVW f ZWWYjZ WVrZp WZabƒ[Z`\ dXZ[Y_VdaWn vV_Ud^_ _UVWXZ[cd[ dX_V _UZ _UZ sYabkVb_U Y`UVZ\Zb ^WVaj daZeWVbZb dXZ[Y_VdaW Y`UVZ\Ys]Z sYabkVb_U VW [d^jU]q UY]c fdc VrY_Vdap _UZ sYabe VW W]dk]q `da\Z[jVaj _dkY[bW _UZ sYabkVb_U dc _UZ kVb_U Y`UVZ\Zb cd[ _kdeWVbZb `d ff ^aV`Y_Vdan WZabƒ[Z`\ _ZW_e`YWZn TUZ[Z VW ad [ZY] bVuZ[Za`Z Va _UZ VW ad [ZY] XZ[cd[ f Ya`Z bVuZ[Za`Z sZ_kZZa XZ[cd[ f Ya`Z kUZ_UZ[ ghi h^_ d[ ghi Z_ VW ^WZbn _UZTUZ[Z _kd Wqa`U[daVrY_Vda f Z`UYaVW f W YaY]qrZbn  dke  dkZ\Z[p _UZ XZ[cd[ f Ya`Z VW ^W^Y]]q W]VjU_]q sZ__Z[ Z\Z[p Vc f Z f d[q VW ad_ Y]]d`Y_Zb ^WVaj _UZ X[d\VbZb kUZa ^WVaj ghi vVa cZa`Z cd[ Wqa`U[daVrVaj _UZ XY[e ghiec^a`_Vdap _UZ XZ[cd[ f Ya`Z ^WVaj Z_ kYW _V`VXY_Vaj X[d`ZWWZW _UYa _UZ XZ[cd[ f Ya`Z kUZa ^WVaj W]VjU_]q sZ__Z[ _UYa _UY_ Y`UVZ\Zb kV_U ghi ghi h^_n ghi vVa ¸_Y[_ƒhdW_ cd[ Wqa`U[daVrY_Vdan TUZ WV_^Y_Vda VW WVf V]Y[ cd[ _UZ Va_Z[eadbZ `YWZn TUZ Vf X]Z f Za_Y_Vda dc _UZ _ZW_eW^V_Z ^WVaj ghi vVa cZa`Z cd[ Wqa`U[daVrY_Vda Y`UVZ\ZW Y Wd f Ze kUY_ sZ__Z[ XZ[cd[ f Ya`Z _UYa _UZ _ZW_e`YWZ ^WVaj ghi vVa ¸_Y[_ƒ hdW_n od[ Y]] _ZW_Wp _UZ ^WYjZ dc tŠ

Two-sided vs. One-sided performance

Two-sided vs. One-sided performance

3500

1000 Send/Recv Put start/post Get start/post Put fence Get fence

3000

Send/Recv Put start/post Get start/post Put fence Get fence Maxmimum Bandwidth [MBytes/sec]

Maxmimum Bandwidth [MBytes/sec]

800 2500

2000

1500

1000

600

400

200 500

0

0 0

0.5

1 Message Length [MBytes]

1.5

2

0

0.5

1 Message Length [MBytes]

1.5

2

oVj^[Z  ³ hZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW sZ_kZZa X[d`ZWWZW da _UZ WY f Z adbZ ]Zc_  Yab da bVuZ[Za_ adbZW [VjU_  kV_U i g ghi Two-sided vs. One-sided performance SUN Fire 6800

Two-sided vs. One-sided performance SUN Fire 6800 using MPI_Alloc_mem

1200

1200 Send/Recv Get fence Put fence Get start/post Put start/post

1000 Maxmimum Bandwidth [MBytes/sec]

Maxmimum Bandwidth [MBytes/sec]

1000

Send/Recv Get fence Put fence Get start/post Put start/post

800

600

400

200

800

600

400

200

0

0 0

0.5

1 Message Length [MBytes]

1.5

2

0

0.5

1 Message Length [MBytes]

1.5

2

oVj^[Z …³ hZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW kV_U ¸  eghi kV_Ud^_ ]Zc_  Yab kV_U [VjU_  ^WVaj ghi „]]d` f Z f Two-sided vs. One-sided performance

Two-sided vs. One-sided performance

500

90 Send/Recv Put start/post Get start/post Put fence Get fence

450

Maxmimum Bandwidth [MBytes/sec]

400 Maxmimum Bandwidth [MBytes/sec]

Send/Recv Put start/post Get start/post Put fence Get fence

80

350 300 250 200 150 100

70 60 50 40 30 20 10

50 0

0 0

0.5

1 Message Length [MBytes]

1.5

2

0

0.5

1 Message Length [MBytes]

1.5

2

oVj^[Z tŠ³ hZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW sZ_kZZa X[d`ZWWZW da _UZ WY f Z adbZ ]Zc_  Yab da bVuZ[Za_ adbZW [VjU_  kV_U µ„ge†nŠn‹ tt



"   %! 
   ia `da_[Y[q _d _kdeWVbZb XdVa_e_deXdVa_ dXZ[Y_VdaWp _UZ Va_Z[cY`Z bZWVjaZb cd[ daZeWVbZb `d ff ^aV`Y_Vda kd^]b Y]]dk cd[ WZ\Z[Y] Va_Z[e f ZWWYjZ dX_Vf VrY_VdaWn ¸Va`Z _UZ bY_Y _[YaWcZ[ dXZ[Y_VdaW ‚^W_ UY\Z _d sZ mae VWUZb Yc_Z[ `]dWVaj _UZ Y``ZWW Yab ZXdW^[Z kVabdkWp Ya Vf X]Z f Za_Y_Vda `d^]b _Y‰Z Yb\Ya_YjZ dc _UZ cY`_p _UY_ V_ UYW _UZ `d f X]Z_Z W`UZb^]Z dc Y]] dajdVaj dXZ[e Y_VdaWn TUVW kd^]b ZaYs]Z cd[ ZY f X]Z dX_Vf VrY_VdaW da _UZ j[d^X ]Z\Z] YW kZ]] YW sZ_kZZa ZY`U XYV[ dc X[d`ZWWZWn ia _UVW W^sWZ`_Vda kZ kd^]b ]V‰Z _d Z\Y]^Y_Z Udk _UZ Vf X]Z f Za_Y_VdaW UYab]Z _UZ ]Y__Z[ W`ZaY[Vdn q _[YaWcZ[[Vaj Y WVaj]Z s^uZ[ Va  h^_ƒ Z_ dXZ[Y_VdaWp kZ `d f XY[Z _UZ ZZ`^_Vda _Vf Z dc _UVW `d ff ^aV`Ye _Vda W`UZb^]Z _d _UZ `YWZp kUZ[Z _UZ WY f Z Y f d^a_ dc bY_Y VW _[YaWcZ[Zb kV_U Y WVaj]Z h^_ƒ Z_ dXZ[Y_Vdan „ Wf Y[_  Vf X]Z f Za_Y_Vda f VjU_ [Z`djaVrZp _UY_ _UZ dXZ[Y_VdaW `Ya sZ f Z[jZb Va_d Y WVaj]Z daZp WVa`Z _UZ _[YaWcZ[Zb bY_Y cd[ f W Y `da_Vj^d^W s^uZ[ da _UZ Wd^[`Z YW kZ]] YW da _UZ bZW_VaY_Vda X[d`ZWWn  daZ dc _UZ YaY]qrZb ghi ]Vs[Y[VZW W^XXd[_Zb _UZ dX_Vf VrY_Vda bZW`[VsZb Va _UZ X[Z\Vd^W XY[Yj[YXUn TUZ _qXV`Y] sZUY\Vd[ dc f dW_ ghi ]Vs[Y[VZW VW WVf V]Y[ _d _UZ [ZW^]_ WUdka ^WVaj ghi h^_ Yab _UZ W_Y[_ƒXdW_ Wqa`U[daVrY_Vda kV_UVa Y WVaj]Z adbZ da _UZ   ˆ ¸ e  Va mj^[Z ttn TUZ `d ff ^aV`Y_Vda YXXZY[W _d UY\Z Y X[d_d`d] WkV_`U Y_ `Z[_YVa f ZWWYjZ ]Zaj_UWp kUV`U ]ZYbW _d Y _Z f Xd[Y[q b[dX Va _UZ Y\YV]Ys]Z sYabe kVb_U bV[Z`_]q Y_ _UZ WkV_`UVaj XdVa_n od[ _UZ f ^]e _VX]Z f ZWWYjZ W`ZaY[Vd YaY]qrZb Va _UVW W^sWZ`_Vdap _UVW WkV_`UVaj XdVa_ VW WUVc_Zb _dkY[bW Y ]Y[jZ[ d\Z[Y]] bY_Y _[YaWcZ[ WVrZ kV_U Va`[ZYWVaj a^ f sZ[ dc _[YaWcZ[ dXZ[Y_VdaWn TUVW `Ya ]ZYb _d XZ[cd[ f Ya`Z Vf X[d\Ze f Za_W cd[ WUd[_ h^_ƒ Z_ dXZ[Y_VdaWp s^_ Y]Wd UYW _UZ ZuZ`_p _UY_ _UZ d\Z[Y]] Y f d^a_ dc bY_Yp kUV`U UYW _d sZ _[YaWcZ[[Zb _d [ZY`U _UZ f YVf ^ f sYabkVb_U dc _UZ WqW_Z f p UYW _d Va`[ZYWZn   
   !       
          TUVW W^sWZ`_Vda Z\Y]^Y_ZWp Udk W_[daj]q _UZ XYWWV\Z WVbZ dc Y daZeWVbZb dXZ[Y_Vda VW Va\d]\Zb Va _UZ bY_Y _[YaWcZ[n TUZ _ZW_ kUV`U kZ `dab^`_Zb VW WVf V]Y[ _d

 

  TUZ XZ[cd[ f Ya`Z [ZW^]_W Y`UVZ\Zb kV_U µ„g Y[Z X[Ze WZa_Zb Va mj^[Z tŠn od[ sd_U b[V\Z[W YaY]qrZbp _UZ sYabkVb_U Y`UVZ\Zb kV_U daZeWVbZb `d ff ^aV`Y_Vda VW `d f XY[Ys]Z _d _UZ WZabƒ[Z`\ XZ[cd[ f Ya`Zn TUZ da]q bVuZ[Za`Z VWp _UY_ _UZ XZY‰ dsWZ[\Zb Va sd_U X[d_d`d]W sZ_kZZa l Yab  ‹ V]dsq_Z f ZWWYjZWp VW Wd f ZkUY_ ]dkZ[p kUV`U VW X[dsYs]q _UZ [ZW^]_ dc `Y`UVaj ZuZ`_Wn TUZ ^WYjZ dc ghi „]]d` f Z f bdZW ad_ ZuZ`_ _UZ XZ[e cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW ^WVaj _UZWZ _kd bZe \V`ZWp Y]_Ud^jU Y``d[bVaj _d _UZ bd`^ f Za_Y_Vda _UZ ^WZ[ f VjU_ X[dm_ c[d f _UVW c^a`_Vda cd[ `Z[_YVa d_UZ[ bZ\V`ZWn   



    


       
        
   ghiel Y]]dkW _UZ ^WYjZ dc Wde`Y]]Zb Xd[_Ys]Z bZe [V\Zb bY_Y_qXZW Va daZeWVbZb dXZ[Y_VdaWn hd[_Ys]Z bY_Y_qXZW Y[Z bZmaZb _d `daWVW_ dc Y WVaj]Z sYWV` bY_Y_qXZp kUZ[Z _UZ bVWX]Y`Z f Za_ sZ_kZZa ZY`U s]d`‰ `Ya sZ ZX[ZWWZb YW Y f ^]_VX]Z dc _UZ Z_Za_ dc _UZ sYe WV` bY_Y_qXZn ¸Va`Z Y WVaj]Z X[d`ZWW VW bZW`[VsVaj _UZ bY_Y_qXZ XYe [Y f Z_Z[ cd[ _UZ Wd^[`Z Yab bZW_VaY_Vda X[d`ZWWp ^WZ[ bZmaZb bY_Y_qXZW c^[_UZ[ Va`[ZYWZ _UZ `d f X]ZV_q dc _UZ Vf X]Z f Za_Y_Vda dc daZeWVbZb dXZ[Y_VdaWp WVa`Z _UZ ghi ]Vs[Y[q UYW _d mab Y `d f XY`_ bZW`[VX_Vda cd[ bZe [V\Zb bY_Y_qXZWp kUV`U `Ya sZ WZa_ Yab X[d`ZWWZb Va _UZ _Y[jZ_ kVabdk n vV_U _UZ Z`ZX_Vda dc µ„gƒghip Y]] YaY]qrZb Vf e X]Z f Za_Y_VdaW c^]]q W^XXd[_Zb Xd[_Ys]Z bY_Y_qXZW da _UZ YaY]qrZb f Y`UVaZWn  daZ dc _UZ Vf X]Z f Za_Ye _VdaW bVW_Vaj^VWU sZ_kZZa Xd[_Ys]Z Yab adaeXd[_Ys]Z bY_Y_qXZWp kUV`U VW ad_ Ya VWW^Z YW ]daj YW _UZ ]Ve s[Y[q VW ^WZb da]q Va Y Ud f djZaZd^W Za\V[da f Za_n TUZ XZ[cd[ f Ya`Z dc daZeWVbZb dXZ[Y_VdaW ^WVaj bZe [V\Zb bY_Y_qXZW kYW da f dW_ X]Y_cd[ f W `d f XY[Ys]Z _d _UZ XZ[cd[ f Ya`Z dc XdVa_e_deXdVa_ dXZ[Y_VdaW ^We Vaj _UZ WY f Z bY_Y_qXZWn µ„gƒghi da]q W^XXd[_W `da_Vj^d^W bZ[V\Zb bY_Y_qXZWn TUVW cY`_ VW Y]Wd f Zae _VdaZb Va _UZ bd`^ f Za_Y_Vda Yab `dam[ f Zb sq d^[ ZXZ[Vf Za_Wn 

tl

UYab]ZW _UZ d\Z[]YXXVaj dc `d ff ^aV`Y_Vda Yab `d f e X^_Y_Vda Z `VZa_]qp cd[ sd_U Va_[Ye Yab Va_Z[eadbZ W`ZaY[VdWn TUVW [ZW^]_ VW WUdka Va _UZ ^XXZ[ XY[_ dc mj^[Z tln ghiƒ¸ p ¸  eghip µ„gƒghi Yab  V_Y`UVeghi Y]] Va_[db^`Zb Y WVjaVm`Ya_ d\Z[UZYb kUZa d\Z[]YXXVaj `d ff ^aV`Y_Vda Yab `d f X^_Y_Vdan „W Y [ZX[ZWZa_Y_V\Z dc _UZWZ ghi ]Vs[Y[VZWp _UZ [Ze W^]_ dc µ„g ƒghi Va _UZ Va_Z[eadbZ `d ff ^aV`Y_Vda VW WUdka Va _UZ ]dkZ[ XY[_ dc mj^[Z tln

Multi-segmented MPI_Put using MPI_Win_Start/Post 14000 1 segment 2 segments 4 segments 8 segments 16 segments

12000

Bandwidth [MB/s]

10000

8000

6000

4000

2000

0 0

0.5

1 Message Length [MBytes]

1.5

2

Overlapping communication and computation on the IBM p690

oVj^[Z tt³ g^]_VX]Z h^_ dXZ[Y_VdaW ^WVaj W_Y[_ƒXdW_ cd[ Wqa`U[daVrY_Vda da _UZ   ˆ ¸ e  n

2 Get winfence inter Get overlap winfence inter Get winfence intra Get overlap winfence intra

Execution time [msec]

1.5

_UZ d\Z[]YX _ZW_ Va _UZ f XX_ZW_ ~…€ _ZW_W^V_Zn TUZ sZa`U f Y[‰ f ZYW^[ZW m[W_ _UZ _Vf Z [Z ² ^V[Zb cd[ _UZ bY_Y _[YaWcZ[ dc Y jV\Za f ZWWYjZ ]Zaj_Un ia _UZ aZ_ W_ZXp V_ bZ_Z[ f VaZW cd[ Y `Z[_YVa `Y]`^]Y_Vda _UZ X[dse ]Z f WVrZp kUV`U _Y‰ZW _UZ WY f Z Y f d^a_ dc _Vf Z YW _UZ bY_Y _[YaWcZ[n TUZ maY] sZa`U f Y[‰ ZZ`^_ZW _UZ bY_Y _[YaWcZ[ Yab _UZ `d f X^_Y_Vda cd[ _UZ bZ_Z[ f VaZb X[ds]Z f WVrZ WVf ^]_YaZd^W]qn ic Ya ghi Vf X]Z f Zae _Y_Vda `Ya ad_ d\Z[]YX _UZ `d ff ^aV`Y_Vda Yab _UZ `d f X^_Y_Vdap _UZ ZZ`^_Vda _Vf Z WUd^]b sZ YXX[dVe f Y_Z]q _kV`Z YW UVjU YW _UZ X^[Z bY_Y _[YaWcZ[p kUV]Z Ya ghi ]Vs[Y[q `YXYs]Z dc [ZY] d\Z[]YX kV]] ZZ`^_Z _UVW sZa`U f Y[‰ [d^jU]q Va _UZ WY f Z _Vf Z YW _UZ X^[Z bY_Y _[YaWcZ[n  Z`^_Vaj _UVW _ZW_ kV_U daZeWVbZb dXZ[Y_VdaW kYW bdaZ sq Va_[db^`Vaj Y `d f X^_Y_Vda Va _UZ XYWWV\Z WVbZ dc _UZ XVajeXdajn ic Y ]Vs[Y[q XZ[cd[ f W kZ]] Va _UVW _ZW_p V_ `Ya sZ b^Z _d _kd [ZYWdaW³ ia `YWZ _UZ daZeWVbZb dXZ[Y_VdaW Y[Z f dbZ]Zb da _dX dc _kdeWVbZb XdVa_e_deXdVa_ `d ff ^aV`Y_Vdap _UZ ]Vs[Y[q VW `YXYs]Z dc bdVaj X[dj[ZWW d^_WVbZ oVj^[Z tl³ ‡\Z[]YXXVaj `d f X^_Y_Vda Yab daZe dc ghi c^a`_VdaW Znjn ^WVaj Y X[dj[ZWW _U[ZYb  n WVbZb `d ff ^aV`Y_Vda kV_U i g ghi ^XXZ[  Yab µ„g ƒghi ]dkZ[  n TUZ ]Vs[Y[q VW _Y‰Vaj Yb\Ya_YjZ dc aY_V\Z W^XXd[_ cd[ h^_ƒ Z_ dXZ[Y_VdaW da Y X]Y_cd[ f n o[d f _UZ ghi ]Vs[Y[VZW YaY]qrZbp da]q i geghi 1

0.5

0

0

0.5

1 Message Length [MBytes]

1.5

2

Overlapping communication and computation with LAM/MPI

70

Get winfence inter Get overlap winfence inter

60

Execution time [msec]

50

40

30

20

10

0

0

t

0.5

1 Message Length [MBytes]

1.5

2


|   RQ 

ghie]Vs[Y[VZW `Yaad_ sZ _d f Y‰Z Z\Z[q_UVaj Z ² ^Y]]q W]dk  p s^_ _d _Y‰Z Yb\Ya_YjZ dc YW f Yaq dX_Vf VrY_Vda XdWWVsV]V_VZW YW XdWWVs]Zn „bbV_VdaY]]qp daZ WUd^]b Y`e ‰adk]ZbjZp _UY_ _UZ ]Vs[Y[q bZ\Z]dXZ[W Va\ZW_Zb U^jZ Zud[_W _d dX_Vf VrZ ghiet c^a`_VdaY]V_qp Zud[_Wp kUV`U f VjU_ sZ Va\ZW_Zb Va ghiel c^a`_VdaW YW Wdda YW _UZWZ WZ`_VdaW Y[Z f d[Z kVbZ]q ^WZbn

ia _UVW XYXZ[ kZ X[ZWZa_Zb d^[ ZXZ[VZa`ZW Yab _UZ XZ[cd[ f Ya`Z dc \Y[Vd^W ghi ]Vs[Y[VZWp kV_U [ZWXZ`_ _d _UZ UYab]Vaj dc bqaY f V`Y]]q `[ZY_Zb `d ff ^aV`Ye _d[W Yab daZeWVbZb `d ff ^aV`Y_Vda dXZ[Y_VdaWn  Yae b]Vaj dc bqaY f V` X[d`ZWW f YaYjZ f Za_ VW W_V]] daZ dc _UZ `UYX_Z[W dc _UZ ghiel WXZ`Vm`Y_Vdap kUV`U VW Vf e X]Z f Za_Zb sq cZk ghi ]Vs[Y[VZW Wd cY[n TUZ [ZW^]_W Y`UVZ\Zb Va d^[ YaY]qWVW WUdk UdkZ\Z[p _UY_ _UZ ^We YjZ dc _UZWZ c^a`_VdaW Y[Z ad_ Vf XdWVaj Yaq XZ[cd[e f Ya`Z XZaY]_VZW _d _UZ Zabe^WZ[ XZ[ WZn ic X[ds]Z f W Y[Z Y[VWVaj kV_U [ZWXZ`_ _d bqaY f V` X[d`ZWW f YaYjZe f Za_p _UZa Y``d[bVaj _d d^[ ZXZ[VZa`Z _UZq Y[Z ad_ [Z]Y_Zb _d _UZV[ Vf X]Z f Za_Y_Vda Va ghin h[ds]Z f W Y[Z ^W^Y]]q `Y^WZb sq _UZ [^a_Vf Z Za\V[da f Za_ Yabƒd[ sY_`U ² ^Z^Z WqW_Z f W Yab _UZV[ [ZW_[V`_V\Z UYab]Vaj dc bqaY f V`Y]]q `[ZY_Zb X[d`ZWWZWn TUZ WZ`_Vda `d\Z[Vaj daZeWVbZb `d ff ^aV`Y_Vda VW W^XXd[_Zb sq f Yaq ghi Vf X]Z f Za_Y_VdaWn TUZ YaY]e qWVW dc _UZV[ XZ[cd[ f Ya`Z [Z\ZY]Wp _UY_ _UZ Wqa`U[de aVrY_Vda `dW_W dc daZeWVbZb dXZ[Y_VdaW Y[Z da Y]] `^[e [Za_ Vf X]Z f Za_Y_VdaW cYV[]q UVjUn TUVW ]ZYbW _d _UZ `da`]^WVdap _UY_ daZeWVbZb dXZ[Y_VdaW WUd^]b ad_ sZ ^WZbp Vc _UZ d\Z[Y]] Y f d^a_ dc bY_Y _[YaWcZ[[Zb sZe _kZZa _kd X[d`ZWWZW VW W f Y]]n TUZ Y`UVZ\Ys]Z sYabe kVb_U kV_U daZeWVbZb dXZ[Y_VdaW VW da Y]] YaY]qrZb X]Y_cd[ f W `]dWZ _d _UZ sYabkVb_U Y`UVZ\Zb kV_U _kde WVbZb `d ff ^aV`Y_Vdan TUZ da]q WV_^Y_Vda kUZ[Z Y XZ[cd[ f Ya`Z sZaZm_ dc daZeWVbZb `d ff ^aV`Y_Vda d\Z[ _kdeWVbZb `d ff ^aV`Y_Vda `d^]b sZ dsWZ[\Zb kYWp Vc _UZ f Z f d[q ^WZb Va _UZ bY_Y _[YaWcZ[ UYb sZZa Y]e ]d`Y_Zb _U[d^jU Y WXZ`VY] ghi c^a`_Vdan TUVW c^a`e _VdaY]V_q dc X[d\VbVaj cYW_  f Z f d[q VW UdkZ\Z[ ad_ kVbZ]q W^XXd[_Zb sq `^[[Za_ ghi ]Vs[Y[VZWn TUZ d\Z[e Y]] `da`]^WVda dc _UVW YaY]qWVW _UZ[Zcd[Z UYW _d sZp _UY_ ^WZ[W WUd^]b ad_ WkV_`U c[d f _kdeWVbZb `d ff ^aV`Ye _Vda _d daZeWVbZb cd[ XZ[cd[ f Ya`Z [ZYWdaWp s^_ da]q Va `YWZW kUZ[Z V_ f Y_`UZW _UZ `d ff ^aV`Y_Vda XY__Z[a dc _UZV[ YXX]V`Y_Vda sZ__Z[ _UYa _kdeWVbZb dXZ[Y_VdaWn TUZ Va_Z[X[Z_Y_Vda dc _UZ [ZW^]_W X[ZWZa_Zb Y[Z _kde cd]b³ da daZ UYabp kZ WZZ Y W_[daj \Y[VYa`Z kV_U [Ze WXZ`_ _d _UZ XZ[cd[ f Ya`Z dc f Yaq ghiel c^a`_VdaWp kUV`U `Ya `dac^WZ _UZ YXX]V`Y_Vda bZ\Z]dXZ[ Yab Vae ¹^Za`Z _UZV[ bZ`VWVda kUZ_UZ[ _d ^WZ ghiel c^a`e _VdaY]V_q d[ ad_n ‡a _UZ d_UZ[ UYabp _UZ jdY] dc

M S  y z   {     y PO TUZ Y^_Ud[W kd^]b ]V‰Z _d _UYa‰ _UZ aV\Z[WV_q dc TZaaZWWZZ ad\V]]Zp _UZ  VjU hZ[cd[ f Ya`Z ˆd f e X^_Vaj ˆZa_Z[ ¸_^__jY[_  µ´¸  p _UZ aV\Z[WV_q dc

] f Yab _UZ od[W`U^ajWrZa_[^ f  ^Z]V`U cd[ X[d\Vbe Vaj _UZV[ f Y`UVaZW ^WZb Va d^[ _ZW_Wn o^[_UZ[ f d[Zp kZ kd^]b ]V‰Z _d Y`‰adk]ZbjZ _UZ W^XXd[_ dc  d]jZ[

Z[jZ[ c[d f   ˆ  ^[dXZn    Q  y S  O

~t€ v n [dXXp  n µ^W‰p  n ¶dWWp „n ¸‰‚Z]]^ f ³                                  p hY[Y]]Z] ˆd f X^_Vajp ll t……   n ~l€ gZWWYjZ hYWWVaj ia_Z[cY`Z od[^ f n               

TZ`UaV`Y] [ZXd[_p  ^aZ t…… n    !"!# " U__X³ƒƒkkk nf XVecd[^ f nd[j ~€ gZWWYjZ hYWWVaj ia_Z[cY`Z od[^ f n  $ %&  

                 ^]q t p t……†n ~‹€ ´d]c ´YsZaWZVcaZ[ Yab „]V`Z  n daVjZW %'  (  ) * + ,   +    - p Va „n dbZp Tn µ^bkVjp ´n vVWW f ^Z]]Z[  bWn p h[d`ZZbVajW dc _UZ  ^[dehY[ lŠŠŠp XXn tl†etl  p ¸X[VajZ[p lŠŠŠn ~ € gY`VZ‚ d]ZsVZkW‰V Yab  ZWXZ[ µY[WWda T[YZu  $ *      .   /        0      ./ p 1 VYaaVW ˆd_[daVWp  Y`‰ ¶dajY[[Y  bWn p ´Z`Za_ „b\Ya`ZW Va hY[Y]]Z] 2 V[_^Y] gYe `UVaZ Yab gZWWYjZ hYWWVaj ia_Z[cY`Z p XXn t elp ¸X[VajZ[p lŠŠtn t‹

~ € ]Zaa µ^Z`‰Zp vZV  ^ % /     d_ZW Va ˆd f X^_Z[ ¸`VZa`Z \d]n l  ‹Šp XXn e…†p       $ *       /       ¸X[VajZ[ lŠŠn

p _Z`UaV`Y] [ZXd[_p ¶Z`Z sZ[ ltp lŠŠlp . 
  ! f v n [dXXp Yab n TddaZa  

U__X³ƒƒkkk nX^s]V`nVYW_Z_ZnZb^ƒ j[]ƒX^s]V`Y_VdaWnU_ f~t‹€ ]  ´n TUY‰^[p    
       *        

~†€ ¸n dd_U Yab  n gd^[Yd                *      .   /    p Va ¶VZ_Z[ [Yar]f ^Z]]Z[p hZ_Z[ Y`W^‰p  Y`‰  n ¶dae            Va X[d`ZZbVajW dc jY[[Y  bWn p ´Z`Za_ „b\Ya`ZW Va hY[Y]]Z] 2 V[_^Y] ¸^XZ[`d f X^_Vaj lŠŠŠp ¶Y]]YWp T p ¸„n gY`UVaZ Yab gZWWYjZ hYWWVaj ia_Z[cY`Z p µZ`_^[Z ~ €  Z[[ f Yaa gVZ[Zabd[up ]YZ[Z ˆYWWV[Z[p Yab  Z]e  d_ZW Va ˆd f X^_Z[ ¸`VZa`Z \d]n l‹tp XXn †e  †p f ^_ ¸`UkY f sd[a   -  ,   +    - ¸X[VajZ[ lŠŠ‹n /           /  Va  Y`‰ ¶dae jY[[Yp hZ_Z[ Y`W^‰p  d[sZ[_ hdbUd[Wr‰V  bWn p ´Z`Za_ „b\Ya`ZW Va hY[Y]]Z] 2 V[_^Y] gY`UVaZ Yab gZWWYjZ hYWWVaj ia_Z[cY`Z p XXn t el  p ¸X[VajZ[p lŠŠŠn ~…€ v n [dXX Yab  n µ^W‰  /    /

               Va  n n ¶dajY[[Yp  nµ^ ² ^Z Yab Tn gY[jY]Zc  bWn p ´Ze `Za_ Yb\Ya`ZW Va hY[Y]]Z] 2 V[_^Y] 2Y`UVaZ Yab gZWWYjZ hYWWVaj ia_Z[cY`Zp XXn ttet p ¸X[VajZ[p t………n ~tŠ€ ´n  n ´Z^WWaZ[p hn ¸YabZ[Wp µn h[Z`UZ]_ Yab g n g ^]]Z[          /   +    - p Va 2 n „]ZYab[d\ Yab  n  n ¶dajY[[Y  bWn p ´Z`Za_ Yb\Ya`ZW Va hY[Y]]Z] 2 V[_^Y] 2 Ye `UVaZ Yab gZWWYjZ hYWWVaj ia_Z[cY`Zp XXn le …p ¸X[VajZ[p t………n ~tt€ ´n  Z f XZ] +             -     
     h[ZWZa_Zb Y_ ¸h  ˆ vd[‰WUdXp v^XXZ[_Y]p Z[ f Yaqp ¸ZX_n t………n ~tl€  n Ys[VZ]p g n ´ZW`U Yab ´n ´^ZU]Z   

     +    -       
       /   Va n  Z[_rsZ[jZ[p „n  dZ‰e W_[Yp ´n vV]]VY f  bWn  VjU hZ[cd[ f Ya`Z ˆd f e X^_Vaj Yab  Z_kd[‰Vaj p XXn ‹…e Šlp ¸X[VajZ[p lŠŠtn ~t€  n Ys[VZ]p n  n oYjj Yab  n  n ¶dajY[[Y % 

/           $ 
    

/                Va  Y`‰  n ¶dajY[[Yp ¶d f ZaV`d µYcd[ZarYp ¸Y]\Y_d[Z ‡[e ]YabZ  bWn p ´Z`Za_ „b\Ya`ZW Va hY[Y]]Z] 2 V[_^Y] gY`UVaZ Yab gZWWYjZ hYWWVa ia_Z[cY`Z p µZ`_^[Z t