| 1 |
/* |
| 2 |
* Copyright (c) 2013. Intel Corporation. All rights reserved. |
| 3 |
* Copyright (c) 2006-2012. QLogic Corporation. All rights reserved. |
| 4 |
* Copyright (c) 2003-2006, PathScale, Inc. All rights reserved. |
| 5 |
* |
| 6 |
* This software is available to you under a choice of one of two |
| 7 |
* licenses. You may choose to be licensed under the terms of the GNU |
| 8 |
* General Public License (GPL) Version 2, available from the file |
| 9 |
* COPYING in the main directory of this source tree, or the |
| 10 |
* OpenIB.org BSD license below: |
| 11 |
* |
| 12 |
* Redistribution and use in source and binary forms, with or |
| 13 |
* without modification, are permitted provided that the following |
| 14 |
* conditions are met: |
| 15 |
* |
| 16 |
* - Redistributions of source code must retain the above |
| 17 |
* copyright notice, this list of conditions and the following |
| 18 |
* disclaimer. |
| 19 |
* |
| 20 |
* - Redistributions in binary form must reproduce the above |
| 21 |
* copyright notice, this list of conditions and the following |
| 22 |
* disclaimer in the documentation and/or other materials |
| 23 |
* provided with the distribution. |
| 24 |
* |
| 25 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 26 |
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 27 |
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 28 |
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 29 |
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 30 |
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 31 |
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 32 |
* SOFTWARE. |
| 33 |
*/ |
| 34 |
|
| 35 |
#include "psm_user.h" |
| 36 |
#include "ipserror.h" |
| 37 |
#include "ips_proto.h" |
| 38 |
#include "ips_proto_internal.h" |
| 39 |
#include <dlfcn.h> |
| 40 |
|
| 41 |
#define DF_OPP_LIBRARY "libofedplus.so" |
| 42 |
#define DATA_VFABRIC_OFFSET 8 |
| 43 |
|
| 44 |
/* SLID and DLID are in network byte order */ |
| 45 |
static psm_error_t |
| 46 |
ips_opp_get_path_rec(ips_path_type_t type, struct ips_proto *proto, |
| 47 |
uint16_t slid, uint16_t dlid, uint16_t desthca_type, |
| 48 |
ips_path_rec_t **path_rec) |
| 49 |
{ |
| 50 |
psm_error_t err = PSM_OK; |
| 51 |
ibta_path_rec_t query; |
| 52 |
ips_opp_path_rec_t *opp_path_rec; |
| 53 |
int opp_err; |
| 54 |
ENTRY elid, *epath = NULL; |
| 55 |
char eplid[128]; |
| 56 |
uint64_t timeout_ack_ms; |
| 57 |
|
| 58 |
/* Query path record query cache first */ |
| 59 |
bzero(&query, sizeof(query)); |
| 60 |
bzero(eplid, sizeof(eplid)); |
| 61 |
|
| 62 |
/* Bulk service ID is control service id + 1 */ |
| 63 |
switch(type) { |
| 64 |
case IPS_PATH_NORMAL_PRIORITY: |
| 65 |
case IPS_PATH_LOW_PRIORITY: |
| 66 |
query.service_id = |
| 67 |
__cpu_to_be64(proto->ep->service_id + DATA_VFABRIC_OFFSET); |
| 68 |
break; |
| 69 |
case IPS_PATH_HIGH_PRIORITY: |
| 70 |
default: |
| 71 |
query.service_id = __cpu_to_be64(proto->ep->service_id); |
| 72 |
} |
| 73 |
|
| 74 |
query.slid = slid; |
| 75 |
query.dlid = dlid; |
| 76 |
|
| 77 |
snprintf(eplid, sizeof(eplid), "%s_%x_%x", (type == IPS_PATH_HIGH_PRIORITY) ? "HIGH" : "LOW", query.slid,query.dlid); |
| 78 |
elid.key = eplid; |
| 79 |
hsearch_r(elid, FIND, &epath, &proto->ips_path_rec_hash); |
| 80 |
|
| 81 |
if (!epath) { /* Unable to find path record in cache */ |
| 82 |
elid.key = psmi_calloc(proto->ep, UNDEFINED, 1, strlen(eplid) + 1); |
| 83 |
opp_path_rec = (ips_opp_path_rec_t*) |
| 84 |
psmi_calloc(proto->ep, UNDEFINED, 1, sizeof(ips_opp_path_rec_t)); |
| 85 |
if (!elid.key || !opp_path_rec) { |
| 86 |
if (elid.key) psmi_free(elid.key); |
| 87 |
if (opp_path_rec) psmi_free(opp_path_rec); |
| 88 |
err = PSM_NO_MEMORY; |
| 89 |
goto fail; |
| 90 |
} |
| 91 |
|
| 92 |
/* Get path record between local LID and remote */ |
| 93 |
opp_err = proto->opp_fn.op_path_get_path_by_rec(proto->opp_ctxt, &query, |
| 94 |
&opp_path_rec->opp_response); |
| 95 |
if (opp_err) { |
| 96 |
psmi_free(opp_path_rec); |
| 97 |
psmi_free(elid.key); |
| 98 |
err = PSM_EPID_PATH_RESOLUTION; |
| 99 |
goto fail; |
| 100 |
} |
| 101 |
|
| 102 |
/* Create path record */ |
| 103 |
opp_path_rec->ips.epr_slid = opp_path_rec->opp_response.slid; |
| 104 |
opp_path_rec->ips.epr_dlid = opp_path_rec->opp_response.dlid; |
| 105 |
opp_path_rec->ips.epr_mtu = |
| 106 |
min(ibta_mtu_enum_to_int(opp_path_rec->opp_response.mtu & 0x3f), |
| 107 |
proto->epinfo.ep_mtu); |
| 108 |
opp_path_rec->ips.epr_pkey = ntohs(opp_path_rec->opp_response.pkey); |
| 109 |
opp_path_rec->ips.epr_sl = ntohs(opp_path_rec->opp_response.qos_class_sl); |
| 110 |
opp_path_rec->ips.epr_static_rate = opp_path_rec->opp_response.rate & 0x3f; |
| 111 |
opp_path_rec->ips.epr_static_ipd = |
| 112 |
proto->ips_ipd_delay[opp_path_rec->ips.epr_static_rate]; |
| 113 |
|
| 114 |
/* Setup CCA parameters for path */ |
| 115 |
if (opp_path_rec->ips.epr_sl > 15) { |
| 116 |
psmi_free(opp_path_rec); |
| 117 |
psmi_free(elid.key); |
| 118 |
err = PSM_INTERNAL_ERR; |
| 119 |
goto fail; |
| 120 |
} |
| 121 |
if (!(proto->ccti_ctrlmap&(1<<opp_path_rec->ips.epr_sl))) { |
| 122 |
_IPATH_CCADBG("No CCA for sl %d, disable CCA\n", |
| 123 |
opp_path_rec->ips.epr_sl); |
| 124 |
proto->flags &= ~IPS_PROTO_FLAG_CCA; |
| 125 |
} |
| 126 |
opp_path_rec->ips.proto = proto; |
| 127 |
opp_path_rec->ips.epr_ccti_min = proto->cace[opp_path_rec->ips.epr_sl].ccti_min; |
| 128 |
opp_path_rec->ips.epr_ccti = opp_path_rec->ips.epr_ccti_min; |
| 129 |
psmi_timer_entry_init(&opp_path_rec->ips.epr_timer_cca, |
| 130 |
ips_cca_timer_callback, &opp_path_rec->ips); |
| 131 |
|
| 132 |
/* Determine active IPD for path. Is max of static rate and CCT table */ |
| 133 |
if ((opp_path_rec->ips.epr_static_ipd) && |
| 134 |
((opp_path_rec->ips.epr_static_ipd + 1) > |
| 135 |
(proto->cct[opp_path_rec->ips.epr_ccti] & CCA_IPD_MASK))) { |
| 136 |
opp_path_rec->ips.epr_active_ipd = opp_path_rec->ips.epr_static_ipd + 1; |
| 137 |
opp_path_rec->ips.epr_cca_divisor = 0; /*Static rate has no CCA divisor */ |
| 138 |
} |
| 139 |
else { |
| 140 |
/* Pick it from the CCT table */ |
| 141 |
opp_path_rec->ips.epr_active_ipd = |
| 142 |
proto->cct[opp_path_rec->ips.epr_ccti] & CCA_IPD_MASK; |
| 143 |
opp_path_rec->ips.epr_cca_divisor = |
| 144 |
proto->cct[opp_path_rec->ips.epr_ccti] >> CCA_DIVISOR_SHIFT; |
| 145 |
} |
| 146 |
|
| 147 |
/* Compute max timeout based on pkt life time for path */ |
| 148 |
timeout_ack_ms = ((4096UL * (1UL << (opp_path_rec->opp_response.pkt_life & 0x3f)))/ 1000000UL); |
| 149 |
opp_path_rec->ips.epr_timeout_ack = |
| 150 |
ms_2_cycles(IPS_PROTO_ERRCHK_MS_MIN_DEFAULT); |
| 151 |
opp_path_rec->ips.epr_timeout_ack_max = |
| 152 |
ms_2_cycles(IPS_PROTO_ERRCHK_MS_MIN_DEFAULT + timeout_ack_ms); |
| 153 |
opp_path_rec->ips.epr_timeout_ack_factor = IPS_PROTO_ERRCHK_FACTOR_DEFAULT; |
| 154 |
|
| 155 |
/* Add path record into cache */ |
| 156 |
strcpy(elid.key, eplid); |
| 157 |
elid.data = (void*) opp_path_rec; |
| 158 |
hsearch_r(elid, ENTER, &epath, &proto->ips_path_rec_hash); |
| 159 |
} |
| 160 |
else /* Path record found in cache */ |
| 161 |
opp_path_rec = (ips_opp_path_rec_t*) epath->data; |
| 162 |
|
| 163 |
/* Dump path record stats */ |
| 164 |
_IPATH_PRDBG("Path Record ServiceID: %"PRIx64" %x -----> %x\n", (uint64_t) __be64_to_cpu(query.service_id), __be16_to_cpu(slid), __be16_to_cpu(dlid)); |
| 165 |
_IPATH_PRDBG("MTU: %x, %x\n", (opp_path_rec->opp_response.mtu & 0x3f), opp_path_rec->ips.epr_mtu); |
| 166 |
_IPATH_PRDBG("PKEY: 0x%04x\n", ntohs(opp_path_rec->opp_response.pkey)); |
| 167 |
_IPATH_PRDBG("SL: 0x%04x\n", ntohs(opp_path_rec->opp_response.qos_class_sl)); |
| 168 |
_IPATH_PRDBG("Rate: %x, IPD: %x\n", (opp_path_rec->opp_response.rate & 0x3f), opp_path_rec->ips.epr_static_ipd); |
| 169 |
_IPATH_PRDBG("Timeout Init.: 0x%"PRIx64" Max: 0x%"PRIx64"\n", opp_path_rec->ips.epr_timeout_ack, opp_path_rec->ips.epr_timeout_ack_max); |
| 170 |
|
| 171 |
/* Return the IPS path record */ |
| 172 |
*path_rec = &opp_path_rec->ips; |
| 173 |
|
| 174 |
fail: |
| 175 |
return err; |
| 176 |
} |
| 177 |
|
| 178 |
static psm_error_t |
| 179 |
ips_opp_path_rec(struct ips_proto *proto, |
| 180 |
uint16_t slid, uint16_t dlid, uint16_t desthca_type, |
| 181 |
unsigned long timeout, |
| 182 |
ips_epaddr_t *ipsaddr) |
| 183 |
{ |
| 184 |
psm_error_t err = PSM_OK; |
| 185 |
uint16_t pidx, cpath, num_path = (1 << proto->epinfo.ep_lmc); |
| 186 |
ips_path_type_t path_type = IPS_PATH_NORMAL_PRIORITY; |
| 187 |
ips_path_rec_t *path; |
| 188 |
uint16_t path_slid, path_dlid; |
| 189 |
psmi_context_t *context = &proto->ep->context; |
| 190 |
|
| 191 |
/* |
| 192 |
* High Priority Path |
| 193 |
* ------------------ |
| 194 |
* |
| 195 |
* Uses the "base" Service ID. For now there exists only 1 high priority |
| 196 |
* path between nodes even for non zero LMC fabrics. |
| 197 |
* TODO: Investigate if there are any benefits for using multiple high |
| 198 |
* priority paths. Initial empirical data shows that this leads to worse |
| 199 |
* performance as the bulk data can induce HOL blocking. |
| 200 |
* Currently the normal and low priority paths are same but at some point |
| 201 |
* we can create separate vFabrics to further distinguish/isolate those |
| 202 |
* traffic flows. |
| 203 |
* |
| 204 |
* Normal/Low Priority Paths |
| 205 |
* ------------------------- |
| 206 |
* |
| 207 |
* Currently these paths are the same i.e. they are queried for the same |
| 208 |
* Service ID/vFabric which is the Base Service ID for High Priority + 1. |
| 209 |
* |
| 210 |
* Use case Scenarios |
| 211 |
* ------------------ |
| 212 |
* |
| 213 |
* Since with vFabrics we have the capability to define different QoS |
| 214 |
* parameters per vFabric it is envisioned that the IPS_PATH_HIGH_PRIORITY is |
| 215 |
* setup in a separate vFabric for high priority traffic. The NORMAL paths |
| 216 |
* are setup in a separate vFabric optimized for high bandwidth. This allows |
| 217 |
* us to potentially have control traffic (RTS, CTS etc.) not be bottlenecked |
| 218 |
* by bulk transfer data. All control messages (ACKs,NAKs, TID_GRANT etc.) |
| 219 |
* also use the high priority control vFabric. |
| 220 |
* |
| 221 |
* NOTE: In order to distinguish between the different vFabrics the user |
| 222 |
* specifies the service ID to use via mpirun (or environment variable). |
| 223 |
* This is the service ID for the high priority control traffic. The bulk |
| 224 |
* data vFabric is identified by service ID + 1. So for each MPI application |
| 225 |
* one should specify two service IDs for the high priority and bulk data. |
| 226 |
* Both these service IDs can be placed in the same vFabric which can be |
| 227 |
* configured for high priority or bandwidth traffic giving us the default |
| 228 |
* behavior upto Infinipath 2.5 release. |
| 229 |
* |
| 230 |
* NOTE: All of the above would have really helped if the S20 silicon could |
| 231 |
* correctly support IBTA QoS features. Due to S20 design we can only have |
| 232 |
* high priority VLarb table (low priority VLarb table results in round |
| 233 |
* robin arbitration ignoring the weights!). But if this is fixed in a |
| 234 |
* subsequent chip respin then this may potentially help our scalability |
| 235 |
* on large fabrics. |
| 236 |
* |
| 237 |
* Mesh/Torus and DOR routed networks |
| 238 |
* ---------------------------------- |
| 239 |
* |
| 240 |
* In a mesh/torus fabric we always have a non zero LMC (atleast 1 can be |
| 241 |
* more). We would like to take advantage of dispersive routing on these |
| 242 |
* fabrics as well to obtain better "worst case/congested" bandwidth. For |
| 243 |
* these networks currently the base LIDs are used for UPDN routing which |
| 244 |
* is suboptimal on these networks. Higher order LIDs (+1 .. +N) use DOR |
| 245 |
* routing (Dimension Ordered Routing) to avoid deadlocks and provide |
| 246 |
* higher performance. If a fabric is disrupted then only the base UPDN |
| 247 |
* routing is available. PSM should continue to operate in this environment |
| 248 |
* albeit with degraded performance. In disrupted fabric the OPP path |
| 249 |
* record queries may fail for some DOR routed LIDs i.e. no path exists |
| 250 |
* PSM should hence ignore path record failures as they indicate a disrupted |
| 251 |
* fabric and only use valid paths that are returned from the replica. This |
| 252 |
* will degenerate to only using the UPDN paths on disrupted fabrics and DOR |
| 253 |
* routes only for fully configured fabrics. Note: For a clean fabric the |
| 254 |
* base LIDs that are configured for UPDN route will not exist in the replica |
| 255 |
* as DOR routes are preferred. Hence we will only dispersively route across |
| 256 |
* the DOR routes only using the UPDN route for disrupted fabrics. |
| 257 |
* |
| 258 |
* AS LONG AS ONE PATH EXISTS (for each of the priorities) COMMUNICATION CAN |
| 259 |
* TAKE PLACE. |
| 260 |
*/ |
| 261 |
|
| 262 |
/* If base lids are only used then reset num_path to 1 */ |
| 263 |
if (proto->flags & IPS_PROTO_FLAG_PPOLICY_STATIC_BASE) |
| 264 |
num_path = 1; |
| 265 |
|
| 266 |
ipsaddr->epr.epr_num_paths[IPS_PATH_HIGH_PRIORITY] = |
| 267 |
ipsaddr->epr.epr_num_paths[IPS_PATH_NORMAL_PRIORITY] = |
| 268 |
ipsaddr->epr.epr_num_paths[IPS_PATH_LOW_PRIORITY] = 0; |
| 269 |
|
| 270 |
/* For now there is always only one high priority path between nodes. */ |
| 271 |
for (pidx = 0,cpath = 0; pidx < num_path && cpath == 0; pidx++) { |
| 272 |
path_slid = __cpu_to_be16(__be16_to_cpu(slid) + pidx); |
| 273 |
path_dlid = __cpu_to_be16(__be16_to_cpu(dlid) + pidx); |
| 274 |
|
| 275 |
err = ips_opp_get_path_rec(IPS_PATH_HIGH_PRIORITY, proto, |
| 276 |
path_slid, path_dlid, |
| 277 |
desthca_type, &path); |
| 278 |
|
| 279 |
if (err == PSM_OK) { /* Valid high priority path found */ |
| 280 |
/* Resolved high priority path successfully */ |
| 281 |
ipsaddr->epr.epr_num_paths[IPS_PATH_HIGH_PRIORITY]++; |
| 282 |
ipsaddr->epr.epr_path[IPS_PATH_HIGH_PRIORITY][cpath] = path; |
| 283 |
|
| 284 |
/* Increment current path index */ |
| 285 |
cpath++; |
| 286 |
} |
| 287 |
} |
| 288 |
|
| 289 |
/* Make sure we have atleast 1 high priority path */ |
| 290 |
if (ipsaddr->epr.epr_num_paths[IPS_PATH_HIGH_PRIORITY] == 0) { |
| 291 |
err = psmi_handle_error(NULL, PSM_EPID_PATH_RESOLUTION, |
| 292 |
"OFEF Plus path lookup failed. Unable to resolve high priority network path for LID 0x%x <---> 0x%x. Is the SM running or service ID %"PRIx64" defined?", ntohs(slid), ntohs(dlid), (uint64_t) proto->ep->service_id); |
| 293 |
goto fail; |
| 294 |
} |
| 295 |
|
| 296 |
/* Next setup the bulk paths. If the subnet administrator has misconfigured |
| 297 |
* or rather not configured two separate service IDs we place the bulk |
| 298 |
* paths in the same vFabric as the control paths. |
| 299 |
*/ |
| 300 |
for (pidx = 0,cpath = 0; pidx < num_path; pidx++) { |
| 301 |
path_slid = __cpu_to_be16(__be16_to_cpu(slid) + pidx); |
| 302 |
path_dlid = __cpu_to_be16(__be16_to_cpu(dlid) + pidx); |
| 303 |
|
| 304 |
retry_path_res: |
| 305 |
err = ips_opp_get_path_rec(path_type, proto, |
| 306 |
path_slid, path_dlid, desthca_type, |
| 307 |
&path); |
| 308 |
if (err != PSM_OK) { |
| 309 |
if (path_type == IPS_PATH_NORMAL_PRIORITY) { |
| 310 |
/* Subnet may only be configured for one service ID/vFabric. Default |
| 311 |
* to using the control vFabric/service ID for bulk data as well. |
| 312 |
*/ |
| 313 |
path_type = IPS_PATH_HIGH_PRIORITY; |
| 314 |
goto retry_path_res; |
| 315 |
} |
| 316 |
|
| 317 |
/* Unable to resolve path for <path_slid, path_dline>. This is possible |
| 318 |
* for disrupted fabrics using DOR routing so continue to acquire paths |
| 319 |
*/ |
| 320 |
err = PSM_OK; |
| 321 |
continue; |
| 322 |
} |
| 323 |
|
| 324 |
/* Valid path. For now both normal and low priority paths are the same */ |
| 325 |
ipsaddr->epr.epr_path[IPS_PATH_NORMAL_PRIORITY][cpath] = path; |
| 326 |
ipsaddr->epr.epr_path[IPS_PATH_LOW_PRIORITY][cpath] = path; |
| 327 |
ipsaddr->epr.epr_num_paths[IPS_PATH_NORMAL_PRIORITY]++; |
| 328 |
ipsaddr->epr.epr_num_paths[IPS_PATH_LOW_PRIORITY]++; |
| 329 |
cpath++; |
| 330 |
} |
| 331 |
|
| 332 |
/* Make sure we have atleast have a single bulk data transfer path */ |
| 333 |
if ((ipsaddr->epr.epr_num_paths[IPS_PATH_NORMAL_PRIORITY] == 0) || |
| 334 |
(ipsaddr->epr.epr_num_paths[IPS_PATH_LOW_PRIORITY] == 0)) { |
| 335 |
err = psmi_handle_error(NULL, PSM_EPID_PATH_RESOLUTION, |
| 336 |
"OFEF Plus path lookup failed. Unable to resolve normal/low priority network path for LID 0x%x <---> 0x%x. Is the SM running or service ID %"PRIx64" defined?", ntohs(slid), ntohs(dlid), (uint64_t) proto->ep->service_id); |
| 337 |
goto fail; |
| 338 |
} |
| 339 |
|
| 340 |
if (proto->flags & IPS_PROTO_FLAG_PPOLICY_ADAPTIVE) { |
| 341 |
ipsaddr->epr.epr_hpp_index = 0; |
| 342 |
ipsaddr->epr.epr_next_path[IPS_PATH_NORMAL_PRIORITY] = |
| 343 |
context->base_info.spi_context % ipsaddr->epr.epr_num_paths[IPS_PATH_NORMAL_PRIORITY]; |
| 344 |
ipsaddr->epr.epr_next_path[IPS_PATH_LOW_PRIORITY] = |
| 345 |
context->base_info.spi_context % ipsaddr->epr.epr_num_paths[IPS_PATH_LOW_PRIORITY]; |
| 346 |
} |
| 347 |
else if (proto->flags & IPS_PROTO_FLAG_PPOLICY_STATIC_DST) |
| 348 |
ipsaddr->epr.epr_hpp_index = |
| 349 |
ipsaddr->epr.epr_context % ipsaddr->epr.epr_num_paths[IPS_PATH_HIGH_PRIORITY]; |
| 350 |
else if (proto->flags & IPS_PROTO_FLAG_PPOLICY_STATIC_SRC) |
| 351 |
ipsaddr->epr.epr_hpp_index = |
| 352 |
context->base_info.spi_context % ipsaddr->epr.epr_num_paths[IPS_PATH_HIGH_PRIORITY]; |
| 353 |
else /* Base LID */ |
| 354 |
ipsaddr->epr.epr_hpp_index = 0; |
| 355 |
|
| 356 |
fail: |
| 357 |
if (err != PSM_OK) |
| 358 |
_IPATH_PRDBG("Unable to get path record for LID 0x%x <---> DLID 0x%x.\n", slid, dlid); |
| 359 |
return err; |
| 360 |
} |
| 361 |
|
| 362 |
static psm_error_t ips_opp_fini(struct ips_proto *proto) |
| 363 |
{ |
| 364 |
psm_error_t err = PSM_OK; |
| 365 |
|
| 366 |
if (proto->opp_lib) |
| 367 |
dlclose(proto->opp_lib); |
| 368 |
|
| 369 |
return err; |
| 370 |
} |
| 371 |
|
| 372 |
psm_error_t ips_opp_init(struct ips_proto *proto) |
| 373 |
{ |
| 374 |
psm_error_t err = PSM_OK; |
| 375 |
struct ipath_base_info *base_info = &proto->ep->context.base_info; |
| 376 |
char hcaName[32]; |
| 377 |
|
| 378 |
proto->opp_lib = dlopen(DF_OPP_LIBRARY, RTLD_NOW); |
| 379 |
if (!proto->opp_lib) { |
| 380 |
char *err = dlerror(); |
| 381 |
_IPATH_ERROR("Unable to open OFED Plus Plus library %s. Error: %s\n", DF_OPP_LIBRARY, |
| 382 |
err ? err : "no dlerror()"); |
| 383 |
goto fail; |
| 384 |
} |
| 385 |
|
| 386 |
/* Resolve symbols that we require within opp library */ |
| 387 |
proto->opp_fn.op_path_find_hca = dlsym(proto->opp_lib, "op_path_find_hca"); |
| 388 |
proto->opp_fn.op_path_open = dlsym(proto->opp_lib, "op_path_open"); |
| 389 |
proto->opp_fn.op_path_close = dlsym(proto->opp_lib, "op_path_close"); |
| 390 |
proto->opp_fn. op_path_get_path_by_rec = dlsym(proto->opp_lib, "op_path_get_path_by_rec"); |
| 391 |
|
| 392 |
/* If we can't resovle any symbol then fail to load opp module */ |
| 393 |
if (!proto->opp_fn.op_path_find_hca || !proto->opp_fn.op_path_open || |
| 394 |
!proto->opp_fn.op_path_close || !proto->opp_fn.op_path_get_path_by_rec) { |
| 395 |
_IPATH_PRDBG("Unable to resolve symbols in OPP library. Unloading.\n"); |
| 396 |
goto fail; |
| 397 |
} |
| 398 |
|
| 399 |
/* If PSM_IDENTIFY is set display the OPP library location being used. */ |
| 400 |
if (getenv("PSM_IDENTIFY")) { |
| 401 |
Dl_info info_opp; |
| 402 |
_IPATH_INFO("PSM path record queries using OFED Plus Plus (%s) from %s\n", |
| 403 |
DF_OPP_LIBRARY, |
| 404 |
dladdr(proto->opp_fn.op_path_open, &info_opp) ? info_opp.dli_fname : |
| 405 |
"Unknown/unsupported version of OPP library found!"); |
| 406 |
} |
| 407 |
|
| 408 |
/* Obtain handle to hca (requires verbs on node) */ |
| 409 |
snprintf(hcaName, sizeof(hcaName), "qib%d", base_info->spi_unit); |
| 410 |
proto->hndl = proto->opp_fn.op_path_find_hca(hcaName, &proto->device); |
| 411 |
if (!proto->hndl) { |
| 412 |
_IPATH_ERROR("OPP: Unable to find HCA %s. Disabling OPP interface for path record queries.\n", hcaName); |
| 413 |
goto fail; |
| 414 |
} |
| 415 |
|
| 416 |
/* Get OPP context */ |
| 417 |
proto->opp_ctxt = proto->opp_fn.op_path_open(proto->device, base_info->spi_port); |
| 418 |
if (!proto->opp_ctxt) { |
| 419 |
_IPATH_ERROR("OPP: Unable to optain OPP context. Disabling OPP interface for path record queries.\n"); |
| 420 |
goto fail; |
| 421 |
} |
| 422 |
|
| 423 |
/* OPP initialized successfully */ |
| 424 |
proto->ibta.get_path_rec = ips_opp_path_rec; |
| 425 |
proto->ibta.fini = ips_opp_fini; |
| 426 |
proto->flags |= IPS_PROTO_FLAG_QUERY_PATH_REC; |
| 427 |
|
| 428 |
return err; |
| 429 |
|
| 430 |
fail: |
| 431 |
_IPATH_ERROR("Make sure SM is running...\n"); |
| 432 |
_IPATH_ERROR("Make sure service dist_sa is running...\n"); |
| 433 |
_IPATH_ERROR("to start dist_sa: service dist_sa start\n"); |
| 434 |
_IPATH_ERROR("or enable it at boot time: iba_config -E dist_sa\n\n"); |
| 435 |
|
| 436 |
err = psmi_handle_error(NULL, PSM_EPID_PATH_RESOLUTION, |
| 437 |
"Unable to initialize OFED Plus library successfully.\n"); |
| 438 |
|
| 439 |
if (proto->opp_lib) |
| 440 |
dlclose(proto->opp_lib); |
| 441 |
|
| 442 |
return err; |
| 443 |
} |
| 444 |
|